From 3fad074417c9a6c9d8bd8c2f015575cd1b864346 Mon Sep 17 00:00:00 2001
From: Henning Andersen <33268011+henningandersen@users.noreply.github.com>
Date: Tue, 28 May 2024 12:57:32 +0200
Subject: [PATCH 001/208] Enable not same executor assertion (#109088)

Enable the assertion introduced in #108934
---
 .../org/elasticsearch/action/support/PlainActionFuture.java    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java
index 938fe4c84480b..c52c9ba1264db 100644
--- a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java
+++ b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java
@@ -407,8 +407,7 @@ public static <T, E extends Exception> T get(CheckedConsumer<PlainActionFuture<T
 
     private boolean assertCompleteAllowed() {
         Thread waiter = sync.getFirstQueuedThread();
-        // todo: reenable assertion once downstream code is updated
-        assert true || waiter == null || allowedExecutors(waiter, Thread.currentThread())
+        assert waiter == null || allowedExecutors(waiter, Thread.currentThread())
             : "cannot complete future on thread "
                 + Thread.currentThread()
                 + " with waiter on thread "

From e5558ca2ab4dc115e3db3e7d524ce128a236f065 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 28 May 2024 07:27:13 -0400
Subject: [PATCH 002/208] ESQL: Prepare analyzer for LOOKUP (#109045)

This extracts two fairly uncontroversial changes that were in the main
LOOKUP PR into a smaller change that's easier to review.
---
 .../elasticsearch/xpack/esql/analysis/Analyzer.java  | 12 +++---------
 .../xpack/esql/parser/ExpressionBuilder.java         |  4 ++--
 .../xpack/esql/parser/LogicalPlanBuilder.java        |  2 +-
 .../xpack/esql/parser/ExpressionTests.java           |  2 +-
 .../xpack/esql/parser/StatementParserTests.java      |  8 ++++----
 5 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index fc4673104a71a..082036e32e71f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -118,16 +118,10 @@ public class Analyzer extends ParameterizedRuleExecutor<LogicalPlan, AnalyzerCon
     private static final Iterable<RuleExecutor.Batch<LogicalPlan>> rules;
 
     static {
-        var resolution = new Batch<>(
-            "Resolution",
-            new ResolveTable(),
-            new ResolveEnrich(),
-            new ResolveFunctions(),
-            new ResolveRefs(),
-            new ImplicitCasting()
-        );
+        var init = new Batch<>("Initialize", Limiter.ONCE, new ResolveTable(), new ResolveEnrich(), new ResolveFunctions());
+        var resolution = new Batch<>("Resolution", new ResolveRefs(), new ImplicitCasting());
         var finish = new Batch<>("Finish Analysis", Limiter.ONCE, new AddImplicitLimit());
-        rules = List.of(resolution, finish);
+        rules = List.of(init, resolution, finish);
     }
 
     private final Verifier verifier;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
index fa4afd9e1e528..c80488cad4190 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
@@ -586,7 +586,7 @@ public Alias visitRenameClause(EsqlBaseParser.RenameClauseContext ctx) {
         NamedExpression newName = visitQualifiedNamePattern(ctx.newName);
         NamedExpression oldName = visitQualifiedNamePattern(ctx.oldName);
         if (newName instanceof UnresolvedNamePattern || oldName instanceof UnresolvedNamePattern) {
-            throw new ParsingException(src, "Using wildcards (*) in RENAME is not allowed [{}]", src.text());
+            throw new ParsingException(src, "Using wildcards [*] in RENAME is not allowed [{}]", src.text());
         }
 
         return new Alias(src, newName.name(), oldName);
@@ -603,7 +603,7 @@ public NamedExpression visitEnrichWithClause(EsqlBaseParser.EnrichWithClauseCont
     private NamedExpression enrichFieldName(EsqlBaseParser.QualifiedNamePatternContext ctx) {
         var name = visitQualifiedNamePattern(ctx);
         if (name instanceof UnresolvedNamePattern up) {
-            throw new ParsingException(source(ctx), "Using wildcards (*) in ENRICH WITH projections is not allowed [{}]", up.pattern());
+            throw new ParsingException(source(ctx), "Using wildcards [*] in ENRICH WITH projections is not allowed [{}]", up.pattern());
         }
         return name;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
index 32c7734193947..a7a3d0b816e79 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
@@ -378,7 +378,7 @@ public PlanFactory visitEnrichCommand(EsqlBaseParser.EnrichCommandContext ctx) {
 
             NamedExpression matchField = ctx.ON() != null ? visitQualifiedNamePattern(ctx.matchField) : new EmptyAttribute(source);
             if (matchField instanceof UnresolvedNamePattern up) {
-                throw new ParsingException(source, "Using wildcards (*) in ENRICH WITH projections is not allowed [{}]", up.pattern());
+                throw new ParsingException(source, "Using wildcards [*] in ENRICH WITH projections is not allowed [{}]", up.pattern());
             }
 
             List<NamedExpression> keepClauses = visitList(this, ctx.enrichWithClause(), NamedExpression.class);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
index 1123b16ee45e8..8853f4661fe9b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
@@ -608,7 +608,7 @@ public void testMultipleProjectPatterns() {
     }
 
     public void testForbidWildcardProjectRename() {
-        assertParsingException(() -> renameExpression("b* AS a*"), "line 1:18: Using wildcards (*) in RENAME is not allowed [b* AS a*]");
+        assertParsingException(() -> renameExpression("b* AS a*"), "line 1:18: Using wildcards [*] in RENAME is not allowed [b* AS a*]");
     }
 
     public void testSimplifyInWithSingleElementList() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
index 568694947e39a..db4d54caf9943 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
@@ -728,15 +728,15 @@ public void testEnrich() {
             processingCommand("enrich _" + mode.name() + ":countries ON country_code")
         );
 
-        expectError("from a | enrich countries on foo* ", "Using wildcards (*) in ENRICH WITH projections is not allowed [foo*]");
-        expectError("from a | enrich countries on foo with bar*", "Using wildcards (*) in ENRICH WITH projections is not allowed [bar*]");
+        expectError("from a | enrich countries on foo* ", "Using wildcards [*] in ENRICH WITH projections is not allowed [foo*]");
+        expectError("from a | enrich countries on foo with bar*", "Using wildcards [*] in ENRICH WITH projections is not allowed [bar*]");
         expectError(
             "from a | enrich countries on foo with x = bar* ",
-            "Using wildcards (*) in ENRICH WITH projections is not allowed [bar*]"
+            "Using wildcards [*] in ENRICH WITH projections is not allowed [bar*]"
         );
         expectError(
             "from a | enrich countries on foo with x* = bar ",
-            "Using wildcards (*) in ENRICH WITH projections is not allowed [x*]"
+            "Using wildcards [*] in ENRICH WITH projections is not allowed [x*]"
         );
         expectError(
             "from a | enrich typo:countries on foo",

From 8329a094ab9fbb98856e18327f435b977055a839 Mon Sep 17 00:00:00 2001
From: Simon Cooper <simon.cooper@elastic.co>
Date: Tue, 28 May 2024 12:46:15 +0100
Subject: [PATCH 003/208] Add permission to secure access to certain config
 files (#107827)

This adds a SecuredFileAccessPermission that Elasticsearch and plugins can use to limit access to certain files, so that only code that also has that same permission can access the specified files
---
 docs/changelog/107827.yaml                    |  5 ++
 .../bootstrap/ESPolicyUnitTests.java          | 87 +++++++++++++------
 .../SecuredFileAccessPermission.java          | 25 ++++++
 .../org/elasticsearch/bootstrap/ESPolicy.java | 80 ++++++++++++++---
 .../elasticsearch/bootstrap/PolicyUtil.java   | 27 +++---
 .../org/elasticsearch/bootstrap/Security.java | 78 +++++++++++++----
 .../bootstrap/BootstrapForTesting.java        | 11 +--
 7 files changed, 239 insertions(+), 74 deletions(-)
 create mode 100644 docs/changelog/107827.yaml
 create mode 100644 server/src/main/java/org/elasticsearch/SecuredFileAccessPermission.java

diff --git a/docs/changelog/107827.yaml b/docs/changelog/107827.yaml
new file mode 100644
index 0000000000000..7cf217567b745
--- /dev/null
+++ b/docs/changelog/107827.yaml
@@ -0,0 +1,5 @@
+pr: 107827
+summary: Add permission to secure access to certain config files
+area: Security
+type: bug
+issues: []
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java
index bd26146f92c0d..86a5e6734c701 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
 
 import java.io.FilePermission;
 import java.net.SocketPermission;
@@ -19,11 +20,15 @@
 import java.security.Permission;
 import java.security.PermissionCollection;
 import java.security.Permissions;
+import java.security.Policy;
 import java.security.ProtectionDomain;
 import java.security.cert.Certificate;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+
+import static java.util.Map.entry;
+import static org.elasticsearch.bootstrap.ESPolicy.POLICY_RESOURCE;
 
 /**
  * Unit tests for ESPolicy: these cannot run with security manager,
@@ -32,6 +37,13 @@
 public class ESPolicyUnitTests extends ESTestCase {
 
     static final Map<String, URL> TEST_CODEBASES = BootstrapForTesting.getCodebases();
+    static Policy DEFAULT_POLICY;
+
+    @BeforeClass
+    public static void setupPolicy() {
+        assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
+        DEFAULT_POLICY = PolicyUtil.readPolicy(ESPolicy.class.getResource(POLICY_RESOURCE), TEST_CODEBASES);
+    }
 
     /**
      * Test policy with null codesource.
@@ -42,12 +54,11 @@ public class ESPolicyUnitTests extends ESTestCase {
      */
     @SuppressForbidden(reason = "to create FilePermission object")
     public void testNullCodeSource() throws Exception {
-        assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
         // create a policy with AllPermission
         Permission all = new AllPermission();
         PermissionCollection allCollection = all.newPermissionCollection();
         allCollection.add(all);
-        ESPolicy policy = new ESPolicy(TEST_CODEBASES, allCollection, Collections.emptyMap(), true, List.of(), List.of());
+        ESPolicy policy = new ESPolicy(DEFAULT_POLICY, allCollection, Map.of(), true, List.of(), Map.of());
         // restrict ourselves to NoPermission
         PermissionCollection noPermissions = new Permissions();
         assertFalse(policy.implies(new ProtectionDomain(null, noPermissions), new FilePermission("foo", "read")));
@@ -58,9 +69,8 @@ public void testNullCodeSource() throws Exception {
      */
     @SuppressForbidden(reason = "to create FilePermission object")
     public void testNullLocation() throws Exception {
-        assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
         PermissionCollection noPermissions = new Permissions();
-        ESPolicy policy = new ESPolicy(TEST_CODEBASES, noPermissions, Collections.emptyMap(), true, List.of(), List.of());
+        ESPolicy policy = new ESPolicy(DEFAULT_POLICY, noPermissions, Map.of(), true, List.of(), Map.of());
         assertFalse(
             policy.implies(
                 new ProtectionDomain(new CodeSource(null, (Certificate[]) null), noPermissions),
@@ -70,9 +80,8 @@ public void testNullLocation() throws Exception {
     }
 
     public void testListen() {
-        assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
         final PermissionCollection noPermissions = new Permissions();
-        final ESPolicy policy = new ESPolicy(TEST_CODEBASES, noPermissions, Collections.emptyMap(), true, List.of(), List.of());
+        final ESPolicy policy = new ESPolicy(DEFAULT_POLICY, noPermissions, Map.of(), true, List.of(), Map.of());
         assertFalse(
             policy.implies(
                 new ProtectionDomain(ESPolicyUnitTests.class.getProtectionDomain().getCodeSource(), noPermissions),
@@ -83,14 +92,13 @@ public void testListen() {
 
     @SuppressForbidden(reason = "to create FilePermission object")
     public void testDataPathPermissionIsChecked() {
-        assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
         final ESPolicy policy = new ESPolicy(
-            TEST_CODEBASES,
+            DEFAULT_POLICY,
             new Permissions(),
-            Collections.emptyMap(),
+            Map.of(),
             true,
             List.of(new FilePermission("/home/elasticsearch/data/-", "read")),
-            List.of()
+            Map.of()
         );
         assertTrue(
             policy.implies(
@@ -101,27 +109,52 @@ public void testDataPathPermissionIsChecked() {
     }
 
     @SuppressForbidden(reason = "to create FilePermission object")
-    public void testForbiddenFilesAreForbidden() {
-        assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
-
-        FilePermission configPerm = new FilePermission("/home/elasticsearch/config/-", "read");
-        PermissionCollection coll = configPerm.newPermissionCollection();
-        coll.add(configPerm);
+    public void testSecuredAccess() {
+        String file1 = "/home/elasticsearch/config/pluginFile1.yml";
+        URL codebase1 = randomFrom(TEST_CODEBASES.values());
+        String file2 = "/home/elasticsearch/config/pluginFile2.yml";
+        URL codebase2 = randomValueOtherThan(codebase1, () -> randomFrom(TEST_CODEBASES.values()));
+        String dir1 = "/home/elasticsearch/config/pluginDir/";
+        URL codebase3 = randomValueOtherThanMany(Set.of(codebase1, codebase2)::contains, () -> randomFrom(TEST_CODEBASES.values()));
+        URL otherCodebase = randomValueOtherThanMany(
+            Set.of(codebase1, codebase2, codebase3)::contains,
+            () -> randomFrom(TEST_CODEBASES.values())
+        );
 
         ESPolicy policy = new ESPolicy(
-            TEST_CODEBASES,
-            coll,
-            Collections.emptyMap(),
+            DEFAULT_POLICY,
+            new Permissions(),
+            Map.of(),
             true,
             List.of(),
-            List.of(new FilePermission("/home/elasticsearch/config/forbidden.yml", "read"))
-        );
-        ProtectionDomain pd = new ProtectionDomain(
-            new CodeSource(randomBoolean() ? null : randomFrom(TEST_CODEBASES.values()), (Certificate[]) null),
-            new Permissions()
+            Map.ofEntries(entry(file1, Set.of(codebase1)), entry(file2, Set.of(codebase1, codebase2)), entry(dir1 + "*", Set.of(codebase3)))
         );
 
-        assertTrue(policy.implies(pd, new FilePermission("/home/elasticsearch/config/config.yml", "read")));
-        assertFalse(policy.implies(pd, new FilePermission("/home/elasticsearch/config/forbidden.yml", "read")));
+        ProtectionDomain nullDomain = new ProtectionDomain(new CodeSource(null, (Certificate[]) null), new Permissions());
+        ProtectionDomain codebase1Domain = new ProtectionDomain(new CodeSource(codebase1, (Certificate[]) null), new Permissions());
+        ProtectionDomain codebase2Domain = new ProtectionDomain(new CodeSource(codebase2, (Certificate[]) null), new Permissions());
+        ProtectionDomain codebase3Domain = new ProtectionDomain(new CodeSource(codebase3, (Certificate[]) null), new Permissions());
+        ProtectionDomain otherCodebaseDomain = new ProtectionDomain(new CodeSource(otherCodebase, (Certificate[]) null), new Permissions());
+
+        Set<String> actions = Set.of("read", "write", "read,write", "delete", "read,write,execute,readlink,delete");
+
+        assertFalse(policy.implies(nullDomain, new FilePermission(file1, randomFrom(actions))));
+        assertFalse(policy.implies(otherCodebaseDomain, new FilePermission(file1, randomFrom(actions))));
+        assertTrue(policy.implies(codebase1Domain, new FilePermission(file1, randomFrom(actions))));
+        assertFalse(policy.implies(codebase2Domain, new FilePermission(file1, randomFrom(actions))));
+        assertFalse(policy.implies(codebase3Domain, new FilePermission(file1, randomFrom(actions))));
+
+        assertFalse(policy.implies(nullDomain, new FilePermission(file2, randomFrom(actions))));
+        assertFalse(policy.implies(otherCodebaseDomain, new FilePermission(file2, randomFrom(actions))));
+        assertTrue(policy.implies(codebase1Domain, new FilePermission(file2, randomFrom(actions))));
+        assertTrue(policy.implies(codebase2Domain, new FilePermission(file2, randomFrom(actions))));
+        assertFalse(policy.implies(codebase3Domain, new FilePermission(file2, randomFrom(actions))));
+
+        String dirFile = dir1 + "file.yml";
+        assertFalse(policy.implies(nullDomain, new FilePermission(dirFile, randomFrom(actions))));
+        assertFalse(policy.implies(otherCodebaseDomain, new FilePermission(dirFile, randomFrom(actions))));
+        assertFalse(policy.implies(codebase1Domain, new FilePermission(dirFile, randomFrom(actions))));
+        assertFalse(policy.implies(codebase2Domain, new FilePermission(dirFile, randomFrom(actions))));
+        assertTrue(policy.implies(codebase3Domain, new FilePermission(dirFile, randomFrom(actions))));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/SecuredFileAccessPermission.java b/server/src/main/java/org/elasticsearch/SecuredFileAccessPermission.java
new file mode 100644
index 0000000000000..3d24a9bc5ddb3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/SecuredFileAccessPermission.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch;
+
+import java.security.BasicPermission;
+
+/**
+ * A permission granted to ensure secured access to a file in the config directory.
+ * <p>
+ * By granting this permission, all code that does not have the same permission on the same file
+ * will be denied all read/write access to that file.
+ * Note that you also need to wrap any access to the secured files in an {@code AccessController.doPrivileged()} block
+ * as Elasticsearch itself is denied access to files secured by plugins.
+ */
+public class SecuredFileAccessPermission extends BasicPermission {
+    public SecuredFileAccessPermission(String path) {
+        super(path, "");
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java b/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
index d349403505311..dbc94ad7812a7 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
@@ -21,10 +21,13 @@
 import java.security.Permissions;
 import java.security.Policy;
 import java.security.ProtectionDomain;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 /** custom policy for union of static and dynamic permissions */
 final class ESPolicy extends Policy {
@@ -34,25 +37,28 @@ final class ESPolicy extends Policy {
     /** limited policy for scripts */
     static final String UNTRUSTED_RESOURCE = "untrusted.policy";
 
+    private static final String ALL_FILE_MASK = "read,readlink,write,delete,execute";
+
     final Policy template;
     final Policy untrusted;
     final Policy system;
     final PermissionCollection dynamic;
     final PermissionCollection dataPathPermission;
-    final PermissionCollection forbiddenFilePermission;
-    final Map<String, Policy> plugins;
+    final Map<URL, Policy> plugins;
+    final PermissionCollection allSecuredFiles;
+    final Map<FilePermission, Set<URL>> securedFiles;
 
+    @SuppressForbidden(reason = "Need to access and check file permissions directly")
     ESPolicy(
-        Map<String, URL> codebases,
+        Policy template,
         PermissionCollection dynamic,
-        Map<String, Policy> plugins,
+        Map<URL, Policy> plugins,
         boolean filterBadDefaults,
         List<FilePermission> dataPathPermissions,
-        List<FilePermission> forbiddenFilePermissions
+        Map<String, Set<URL>> securedFiles
     ) {
-        this.template = PolicyUtil.readPolicy(getClass().getResource(POLICY_RESOURCE), codebases);
+        this.template = template;
         this.dataPathPermission = createPermission(dataPathPermissions);
-        this.forbiddenFilePermission = createPermission(forbiddenFilePermissions);
         this.untrusted = PolicyUtil.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), Collections.emptyMap());
         if (filterBadDefaults) {
             this.system = new SystemPolicy(Policy.getPolicy());
@@ -61,6 +67,27 @@ final class ESPolicy extends Policy {
         }
         this.dynamic = dynamic;
         this.plugins = plugins;
+
+        this.securedFiles = securedFiles.entrySet()
+            .stream()
+            .collect(Collectors.toUnmodifiableMap(e -> new FilePermission(e.getKey(), ALL_FILE_MASK), e -> Set.copyOf(e.getValue())));
+        this.allSecuredFiles = createPermission(this.securedFiles.keySet());
+    }
+
+    private static PermissionCollection createPermission(Collection<FilePermission> permissions) {
+        PermissionCollection coll;
+        var it = permissions.iterator();
+        if (it.hasNext() == false) {
+            coll = new Permissions();
+        } else {
+            Permission p = it.next();
+            coll = p.newPermissionCollection();
+            coll.add(p);
+            it.forEachRemaining(coll::add);
+        }
+
+        coll.setReadOnly();
+        return coll;
     }
 
     private static PermissionCollection createPermission(List<FilePermission> permissions) {
@@ -87,12 +114,18 @@ public boolean implies(ProtectionDomain domain, Permission permission) {
             return false;
         }
 
-        // completely deny access to specific files that are forbidden
-        if (forbiddenFilePermission.implies(permission)) {
-            return false;
+        URL location = codeSource.getLocation();
+        if (allSecuredFiles.implies(permission)) {
+            /*
+             * Check if location can access this secured file
+             * The permission this is generated from, SecuredFileAccessPermission, doesn't have a mask,
+             * it just grants all access (and so disallows all access from others)
+             * It's helpful to use the infrastructure around FilePermission here to do the directory structure check with implies
+             * so we use ALL_FILE_MASK mask to check if we can do something with this file, whatever the actual operation we're requesting
+             */
+            return canAccessSecuredFile(location, new FilePermission(permission.getName(), ALL_FILE_MASK));
         }
 
-        URL location = codeSource.getLocation();
         if (location != null) {
             // run scripts with limited permissions
             if (BootstrapInfo.UNTRUSTED_CODEBASE.equals(location.getFile())) {
@@ -100,7 +133,7 @@ public boolean implies(ProtectionDomain domain, Permission permission) {
             }
             // check for an additional plugin permission: plugin policy is
             // only consulted for its codesources.
-            Policy plugin = plugins.get(location.getFile());
+            Policy plugin = plugins.get(location);
             if (plugin != null && plugin.implies(domain, permission)) {
                 return true;
             }
@@ -122,6 +155,29 @@ public boolean implies(ProtectionDomain domain, Permission permission) {
         return template.implies(domain, permission) || dynamic.implies(permission) || system.implies(domain, permission);
     }
 
+    @SuppressForbidden(reason = "We get given an URL by the security infrastructure")
+    private boolean canAccessSecuredFile(URL location, FilePermission permission) {
+        if (location == null) {
+            return false;
+        }
+
+        // check the source
+        Set<URL> accessibleSources = securedFiles.get(permission);
+        if (accessibleSources != null) {
+            // simple case - single-file referenced directly
+            return accessibleSources.contains(location);
+        } else {
+            // there's a directory reference in there somewhere
+            // do a manual search :(
+            // there may be several permissions that potentially match,
+            // grant access if any of them cover this file
+            return securedFiles.entrySet()
+                .stream()
+                .filter(e -> e.getKey().implies(permission))
+                .anyMatch(e -> e.getValue().contains(location));
+        }
+    }
+
     private static void hadoopHack() {
         for (StackTraceElement element : Thread.currentThread().getStackTrace()) {
             if ("org.apache.hadoop.util.Shell".equals(element.getClassName()) && "runCommand".equals(element.getMethodName())) {
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java b/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java
index 5d34a86c2e30b..3279bc5b1bfdf 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/PolicyUtil.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.bootstrap;
 
+import org.elasticsearch.SecuredFileAccessPermission;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
@@ -50,6 +51,7 @@
 import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.management.MBeanPermission;
 import javax.management.MBeanServerPermission;
@@ -60,6 +62,8 @@
 import javax.security.auth.kerberos.DelegationPermission;
 import javax.security.auth.kerberos.ServicePermission;
 
+import static java.util.Map.entry;
+
 public class PolicyUtil {
 
     // this object is checked by reference, so the value in the list does not matter
@@ -158,20 +162,15 @@ public boolean test(Permission permission) {
         // is used to mean names are accepted. We do not use this model for all permissions because many permission
         // classes have their own meaning for some form of wildcard matching of the name, which we want to delegate
         // to those permissions if possible.
-        Map<String, List<String>> classPermissions = Map.of(
-            URLPermission.class,
-            ALLOW_ALL_NAMES,
-            DelegationPermission.class,
-            ALLOW_ALL_NAMES,
-            ServicePermission.class,
-            ALLOW_ALL_NAMES,
-            PrivateCredentialPermission.class,
-            ALLOW_ALL_NAMES,
-            SQLPermission.class,
-            List.of("callAbort", "setNetworkTimeout"),
-            ClassPermission.class,
-            ALLOW_ALL_NAMES
-        ).entrySet().stream().collect(Collectors.toMap(e -> e.getKey().getCanonicalName(), Map.Entry::getValue));
+        Map<String, List<String>> classPermissions = Stream.of(
+            entry(URLPermission.class, ALLOW_ALL_NAMES),
+            entry(DelegationPermission.class, ALLOW_ALL_NAMES),
+            entry(ServicePermission.class, ALLOW_ALL_NAMES),
+            entry(PrivateCredentialPermission.class, ALLOW_ALL_NAMES),
+            entry(SQLPermission.class, List.of("callAbort", "setNetworkTimeout")),
+            entry(ClassPermission.class, ALLOW_ALL_NAMES),
+            entry(SecuredFileAccessPermission.class, ALLOW_ALL_NAMES)
+        ).collect(Collectors.toMap(e -> e.getKey().getCanonicalName(), Map.Entry::getValue));
         PermissionCollection pluginPermissionCollection = new Permissions();
         namedPermissions.forEach(pluginPermissionCollection::add);
         pluginPermissionCollection.setReadOnly();
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Security.java b/server/src/main/java/org/elasticsearch/bootstrap/Security.java
index 1c37b3492c4cb..e24e13dfff372 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/Security.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/Security.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.bootstrap;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.SecuredFileAccessPermission;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
@@ -33,8 +34,11 @@
 import java.nio.file.Files;
 import java.nio.file.NotDirectoryException;
 import java.nio.file.Path;
+import java.security.Permission;
 import java.security.Permissions;
 import java.security.Policy;
+import java.security.UnresolvedPermission;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -42,8 +46,10 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Consumer;
+import java.util.stream.Stream;
 
 import static java.lang.invoke.MethodType.methodType;
+import static org.elasticsearch.bootstrap.ESPolicy.POLICY_RESOURCE;
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath;
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addSingleFilePath;
 import static org.elasticsearch.reservedstate.service.FileSettingsService.OPERATOR_DIRECTORY;
@@ -116,17 +122,18 @@ static void setSecurityManager(@SuppressWarnings("removal") SecurityManager sm)
      * @param filterBadDefaults true if we should filter out bad java defaults in the system policy.
      */
     static void configure(Environment environment, boolean filterBadDefaults, Path pidFile) throws IOException {
-
         // enable security policy: union of template and environment-based paths, and possibly plugin permissions
         Map<String, URL> codebases = PolicyUtil.getCodebaseJarMap(JarHell.parseModulesAndClassPath());
+        Policy mainPolicy = PolicyUtil.readPolicy(ESPolicy.class.getResource(POLICY_RESOURCE), codebases);
+        Map<URL, Policy> pluginPolicies = getPluginAndModulePermissions(environment);
         Policy.setPolicy(
             new ESPolicy(
-                codebases,
+                mainPolicy,
                 createPermissions(environment, pidFile),
-                getPluginAndModulePermissions(environment),
+                pluginPolicies,
                 filterBadDefaults,
                 createRecursiveDataPathPermission(environment),
-                createForbiddenFilePermissions(environment)
+                readSecuredFiles(environment, mainPolicy, codebases.values(), pluginPolicies)
             )
         );
 
@@ -146,8 +153,8 @@ static void configure(Environment environment, boolean filterBadDefaults, Path p
      * we look for matching plugins and set URLs to fit
      */
     @SuppressForbidden(reason = "proper use of URL")
-    static Map<String, Policy> getPluginAndModulePermissions(Environment environment) throws IOException {
-        Map<String, Policy> map = new HashMap<>();
+    static Map<URL, Policy> getPluginAndModulePermissions(Environment environment) throws IOException {
+        Map<URL, Policy> map = new HashMap<>();
         Consumer<PluginPolicyInfo> addPolicy = pluginPolicy -> {
             if (pluginPolicy == null) {
                 return;
@@ -155,7 +162,7 @@ static Map<String, Policy> getPluginAndModulePermissions(Environment environment
 
             // consult this policy for each of the plugin's jars:
             for (URL jar : pluginPolicy.jars()) {
-                if (map.put(jar.getFile(), pluginPolicy.policy()) != null) {
+                if (map.put(jar, pluginPolicy.policy()) != null) {
                     // just be paranoid ok?
                     throw new IllegalStateException("per-plugin permissions already granted for jar file: " + jar);
                 }
@@ -189,16 +196,55 @@ private static List<FilePermission> createRecursiveDataPathPermission(Environmen
         return toFilePermissions(policy);
     }
 
-    private static List<FilePermission> createForbiddenFilePermissions(Environment environment) throws IOException {
-        Permissions policy = new Permissions();
-        addSingleFilePath(policy, environment.configFile().resolve("elasticsearch.yml"), "read,readlink,write,delete,execute");
-        addSingleFilePath(policy, environment.configFile().resolve("jvm.options"), "read,readlink,write,delete,execute");
-        Path jvmOptionsD = environment.configFile().resolve("jvm.options.d");
-        if (Files.isDirectory(jvmOptionsD)) {
-            // we don't want to create this if it doesn't exist
-            addDirectoryPath(policy, "forbidden_access", jvmOptionsD, "read,readlink,write,delete,execute", false);
+    private static Map<String, Set<URL>> readSecuredFiles(
+        Environment environment,
+        Policy template,
+        Collection<URL> mainCodebases,
+        Map<URL, Policy> pluginPolicies
+    ) throws IOException {
+        Map<String, Set<URL>> securedFiles = new HashMap<>();
+
+        for (URL url : mainCodebases) {
+            PolicyUtil.getPolicyPermissions(url, template, environment.tmpFile())
+                .stream()
+                .flatMap(Security::extractSecuredFileName)
+                .map(environment.configFile()::resolve)
+                .forEach(f -> securedFiles.computeIfAbsent(f.toString(), k -> new HashSet<>()).add(url));
         }
-        return toFilePermissions(policy);
+
+        for (var pp : pluginPolicies.entrySet()) {
+            PolicyUtil.getPolicyPermissions(pp.getKey(), pp.getValue(), environment.tmpFile())
+                .stream()
+                .flatMap(Security::extractSecuredFileName)
+                .map(environment.configFile()::resolve)
+                .forEach(f -> securedFiles.computeIfAbsent(f.toString(), k -> new HashSet<>()).add(pp.getKey()));
+        }
+
+        // always add some config files as exclusive files that no one can access
+        // there's no reason for anyone to read these once the security manager is initialized
+        // so if something has tried to grant itself access, crash out with an error
+        addSpeciallySecuredFile(securedFiles, environment.configFile().resolve("elasticsearch.yml").toString());
+        addSpeciallySecuredFile(securedFiles, environment.configFile().resolve("jvm.options").toString());
+        addSpeciallySecuredFile(securedFiles, environment.configFile().resolve("jvm.options.d/-").toString());
+
+        return Collections.unmodifiableMap(securedFiles);
+    }
+
+    private static void addSpeciallySecuredFile(Map<String, Set<URL>> securedFiles, String path) {
+        Set<URL> attemptedToGrant = securedFiles.put(path, Set.of());
+        if (attemptedToGrant != null) {
+            throw new IllegalStateException(attemptedToGrant + " tried to grant access to special config file " + path);
+        }
+    }
+
+    private static Stream<String> extractSecuredFileName(Permission p) {
+        if (p instanceof SecuredFileAccessPermission) {
+            return Stream.of(p.getName());
+        }
+        if (p instanceof UnresolvedPermission up && up.getUnresolvedType().equals(SecuredFileAccessPermission.class.getCanonicalName())) {
+            return Stream.of(up.getUnresolvedName());
+        }
+        return Stream.empty();
     }
 
     /** Adds access to classpath jars/classes for jar hell scan, etc */
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
index f23048185876b..8a3f36ebb1f8a 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
@@ -56,6 +56,7 @@
 import java.util.stream.Collectors;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean;
+import static org.elasticsearch.bootstrap.ESPolicy.POLICY_RESOURCE;
 import static org.elasticsearch.bootstrap.FilePermissionUtils.addDirectoryPath;
 
 /**
@@ -170,12 +171,12 @@ public class BootstrapForTesting {
                 addDirectoryPath(fastPathPermissions, "java.io.tmpdir-fastpath", javaTmpDir, "read,readlink,write,delete", true);
 
                 final Policy esPolicy = new ESPolicy(
-                    codebases,
+                    PolicyUtil.readPolicy(ESPolicy.class.getResource(POLICY_RESOURCE), codebases),
                     perms,
                     getPluginPermissions(),
                     true,
                     Security.toFilePermissions(fastPathPermissions),
-                    List.of()
+                    Map.of()
                 );
                 Policy.setPolicy(new Policy() {
                     @Override
@@ -250,7 +251,7 @@ private static void addClassCodebase(Map<String, URL> codebases, String name, St
      * like core, test-framework, etc. this way tests fail if accesscontroller blocks are missing.
      */
     @SuppressForbidden(reason = "accesses fully qualified URLs to configure security")
-    static Map<String, Policy> getPluginPermissions() throws Exception {
+    static Map<URL, Policy> getPluginPermissions() throws Exception {
         List<URL> pluginPolicies = Collections.list(
             BootstrapForTesting.class.getClassLoader().getResources(PluginDescriptor.ES_PLUGIN_POLICY)
         );
@@ -302,9 +303,9 @@ static Map<String, Policy> getPluginPermissions() throws Exception {
         }
 
         // consult each policy file for those codebases
-        Map<String, Policy> map = new HashMap<>();
+        Map<URL, Policy> map = new HashMap<>();
         for (URL url : codebases) {
-            map.put(url.getFile(), new Policy() {
+            map.put(url, new Policy() {
                 @Override
                 public boolean implies(ProtectionDomain domain, Permission permission) {
                     // implements union

From 0e8847f7d3c71b6121680e36259c723b46d5f4ee Mon Sep 17 00:00:00 2001
From: Jedr Blaszyk <jedrazb@gmail.com>
Date: Tue, 28 May 2024 14:54:07 +0200
Subject: [PATCH 004/208] [Connector API] Remove deprecated feature fields
 (#109074)

---
 .../connector/ConnectorFeatures.java          | 57 ++-----------------
 .../connector/ConnectorFeaturesTests.java     |  8 +--
 .../connector/ConnectorTestUtils.java         |  2 -
 .../application/connector/ConnectorTests.java |  1 -
 4 files changed, 7 insertions(+), 61 deletions(-)

diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java
index bbb8805de1f0f..1b2e7209e41e5 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java
@@ -35,10 +35,6 @@ public class ConnectorFeatures implements Writeable, ToXContentObject {
     @Nullable
     private final FeatureEnabled documentLevelSecurityEnabled;
     @Nullable
-    private final Boolean filteringAdvancedConfigEnabled;
-    @Nullable
-    private final Boolean filteringRulesEnabled;
-    @Nullable
     private final FeatureEnabled incrementalSyncEnabled;
     @Nullable
     private final FeatureEnabled nativeConnectorAPIKeysEnabled;
@@ -49,23 +45,17 @@ public class ConnectorFeatures implements Writeable, ToXContentObject {
      * Constructs a new instance of ConnectorFeatures.
      *
      * @param documentLevelSecurityEnabled  A flag indicating whether document-level security is enabled.
-     * @param filteringAdvancedConfig       A flag indicating whether advanced filtering configuration is enabled.
-     * @param filteringRules                A flag indicating whether filtering rules are enabled.
      * @param incrementalSyncEnabled        A flag indicating whether incremental sync is enabled.
      * @param nativeConnectorAPIKeysEnabled A flag indicating whether support for api keys is enabled for native connectors.
      * @param syncRulesFeatures             An {@link SyncRulesFeatures} object indicating if basic and advanced sync rules are enabled.
      */
     private ConnectorFeatures(
         FeatureEnabled documentLevelSecurityEnabled,
-        Boolean filteringAdvancedConfig,
-        Boolean filteringRules,
         FeatureEnabled incrementalSyncEnabled,
         FeatureEnabled nativeConnectorAPIKeysEnabled,
         SyncRulesFeatures syncRulesFeatures
     ) {
         this.documentLevelSecurityEnabled = documentLevelSecurityEnabled;
-        this.filteringAdvancedConfigEnabled = filteringAdvancedConfig;
-        this.filteringRulesEnabled = filteringRules;
         this.incrementalSyncEnabled = incrementalSyncEnabled;
         this.nativeConnectorAPIKeysEnabled = nativeConnectorAPIKeysEnabled;
         this.syncRulesFeatures = syncRulesFeatures;
@@ -73,16 +63,12 @@ private ConnectorFeatures(
 
     public ConnectorFeatures(StreamInput in) throws IOException {
         this.documentLevelSecurityEnabled = in.readOptionalWriteable(FeatureEnabled::new);
-        this.filteringAdvancedConfigEnabled = in.readOptionalBoolean();
-        this.filteringRulesEnabled = in.readOptionalBoolean();
         this.incrementalSyncEnabled = in.readOptionalWriteable(FeatureEnabled::new);
         this.nativeConnectorAPIKeysEnabled = in.readOptionalWriteable(FeatureEnabled::new);
         this.syncRulesFeatures = in.readOptionalWriteable(SyncRulesFeatures::new);
     }
 
     private static final ParseField DOCUMENT_LEVEL_SECURITY_ENABLED_FIELD = new ParseField("document_level_security");
-    private static final ParseField FILTERING_ADVANCED_CONFIG_ENABLED_FIELD = new ParseField("filtering_advanced_config");
-    private static final ParseField FILTERING_RULES_ENABLED_FIELD = new ParseField("filtering_rules");
     private static final ParseField INCREMENTAL_SYNC_ENABLED_FIELD = new ParseField("incremental_sync");
     private static final ParseField NATIVE_CONNECTOR_API_KEYS_ENABLED_FIELD = new ParseField("native_connector_api_keys");
     private static final ParseField SYNC_RULES_FIELD = new ParseField("sync_rules");
@@ -91,18 +77,14 @@ public ConnectorFeatures(StreamInput in) throws IOException {
         "connector_features",
         true,
         args -> new Builder().setDocumentLevelSecurityEnabled((FeatureEnabled) args[0])
-            .setFilteringAdvancedConfig((Boolean) args[1])
-            .setFilteringRules((Boolean) args[2])
-            .setIncrementalSyncEnabled((FeatureEnabled) args[3])
-            .setNativeConnectorAPIKeysEnabled((FeatureEnabled) args[4])
-            .setSyncRulesFeatures((SyncRulesFeatures) args[5])
+            .setIncrementalSyncEnabled((FeatureEnabled) args[1])
+            .setNativeConnectorAPIKeysEnabled((FeatureEnabled) args[2])
+            .setSyncRulesFeatures((SyncRulesFeatures) args[3])
             .build()
     );
 
     static {
         PARSER.declareObject(optionalConstructorArg(), (p, c) -> FeatureEnabled.fromXContent(p), DOCUMENT_LEVEL_SECURITY_ENABLED_FIELD);
-        PARSER.declareBoolean(optionalConstructorArg(), FILTERING_ADVANCED_CONFIG_ENABLED_FIELD);
-        PARSER.declareBoolean(optionalConstructorArg(), FILTERING_RULES_ENABLED_FIELD);
         PARSER.declareObject(optionalConstructorArg(), (p, c) -> FeatureEnabled.fromXContent(p), INCREMENTAL_SYNC_ENABLED_FIELD);
         PARSER.declareObject(optionalConstructorArg(), (p, c) -> FeatureEnabled.fromXContent(p), NATIVE_CONNECTOR_API_KEYS_ENABLED_FIELD);
         PARSER.declareObject(optionalConstructorArg(), (p, c) -> SyncRulesFeatures.fromXContent(p), SYNC_RULES_FIELD);
@@ -127,12 +109,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             if (documentLevelSecurityEnabled != null) {
                 builder.field(DOCUMENT_LEVEL_SECURITY_ENABLED_FIELD.getPreferredName(), documentLevelSecurityEnabled);
             }
-            if (filteringAdvancedConfigEnabled != null) {
-                builder.field(FILTERING_ADVANCED_CONFIG_ENABLED_FIELD.getPreferredName(), filteringAdvancedConfigEnabled);
-            }
-            if (filteringRulesEnabled != null) {
-                builder.field(FILTERING_RULES_ENABLED_FIELD.getPreferredName(), filteringRulesEnabled);
-            }
             if (incrementalSyncEnabled != null) {
                 builder.field(INCREMENTAL_SYNC_ENABLED_FIELD.getPreferredName(), incrementalSyncEnabled);
             }
@@ -150,8 +126,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(documentLevelSecurityEnabled);
-        out.writeOptionalBoolean(filteringAdvancedConfigEnabled);
-        out.writeOptionalBoolean(filteringRulesEnabled);
         out.writeOptionalWriteable(incrementalSyncEnabled);
         out.writeOptionalWriteable(nativeConnectorAPIKeysEnabled);
         out.writeOptionalWriteable(syncRulesFeatures);
@@ -163,8 +137,6 @@ public boolean equals(Object o) {
         if (o == null || getClass() != o.getClass()) return false;
         ConnectorFeatures features = (ConnectorFeatures) o;
         return Objects.equals(documentLevelSecurityEnabled, features.documentLevelSecurityEnabled)
-            && Objects.equals(filteringAdvancedConfigEnabled, features.filteringAdvancedConfigEnabled)
-            && Objects.equals(filteringRulesEnabled, features.filteringRulesEnabled)
             && Objects.equals(incrementalSyncEnabled, features.incrementalSyncEnabled)
             && Objects.equals(nativeConnectorAPIKeysEnabled, features.nativeConnectorAPIKeysEnabled)
             && Objects.equals(syncRulesFeatures, features.syncRulesFeatures);
@@ -172,21 +144,12 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(
-            documentLevelSecurityEnabled,
-            filteringAdvancedConfigEnabled,
-            filteringRulesEnabled,
-            incrementalSyncEnabled,
-            nativeConnectorAPIKeysEnabled,
-            syncRulesFeatures
-        );
+        return Objects.hash(documentLevelSecurityEnabled, incrementalSyncEnabled, nativeConnectorAPIKeysEnabled, syncRulesFeatures);
     }
 
     public static class Builder {
 
         private FeatureEnabled documentLevelSecurityEnabled;
-        private Boolean filteringAdvancedConfig;
-        private Boolean filteringRules;
         private FeatureEnabled incrementalSyncEnabled;
         private FeatureEnabled nativeConnectorAPIKeysEnabled;
         private SyncRulesFeatures syncRulesFeatures;
@@ -196,16 +159,6 @@ public Builder setDocumentLevelSecurityEnabled(FeatureEnabled documentLevelSecur
             return this;
         }
 
-        public Builder setFilteringAdvancedConfig(Boolean filteringAdvancedConfig) {
-            this.filteringAdvancedConfig = filteringAdvancedConfig;
-            return this;
-        }
-
-        public Builder setFilteringRules(Boolean filteringRules) {
-            this.filteringRules = filteringRules;
-            return this;
-        }
-
         public Builder setIncrementalSyncEnabled(FeatureEnabled incrementalSyncEnabled) {
             this.incrementalSyncEnabled = incrementalSyncEnabled;
             return this;
@@ -224,8 +177,6 @@ public Builder setSyncRulesFeatures(SyncRulesFeatures syncRulesFeatures) {
         public ConnectorFeatures build() {
             return new ConnectorFeatures(
                 documentLevelSecurityEnabled,
-                filteringAdvancedConfig,
-                filteringRules,
                 incrementalSyncEnabled,
                 nativeConnectorAPIKeysEnabled,
                 syncRulesFeatures
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorFeaturesTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorFeaturesTests.java
index 941d0a9ed4594..57f51a1054393 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorFeaturesTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorFeaturesTests.java
@@ -52,7 +52,6 @@ public void testToXContent() throws IOException {
                     "document_level_security": {
                         "enabled": true
                     },
-                    "filtering_advanced_config": true,
                     "sync_rules": {
                         "advanced": {
                             "enabled": false
@@ -70,7 +69,6 @@ public void testToXContent() throws IOException {
     public void testToXContentMissingDocumentLevelSecurity() throws IOException {
         String content = XContentHelper.stripWhitespace("""
                 {
-                    "filtering_advanced_config": true,
                     "sync_rules": {
                         "advanced": {
                             "enabled": false
@@ -88,7 +86,9 @@ public void testToXContentMissingDocumentLevelSecurity() throws IOException {
     public void testToXContentMissingSyncRules() throws IOException {
         String content = XContentHelper.stripWhitespace("""
                 {
-                    "filtering_advanced_config": true
+                    "document_level_security": {
+                        "enabled": true
+                    }
                 }
             """);
 
@@ -98,7 +98,6 @@ public void testToXContentMissingSyncRules() throws IOException {
     public void testToXContentMissingSyncRulesAdvanced() throws IOException {
         String content = XContentHelper.stripWhitespace("""
                 {
-                    "filtering_advanced_config": true,
                     "sync_rules": {
                         "basic": {
                             "enabled": true
@@ -116,7 +115,6 @@ public void testToXContent_NativeConnectorAPIKeysEnabled() throws IOException {
                     "document_level_security": {
                         "enabled": true
                     },
-                    "filtering_advanced_config": true,
                     "sync_rules": {
                         "advanced": {
                             "enabled": false
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java
index b6d45fff1e520..230de44a8f6c5 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java
@@ -156,8 +156,6 @@ public static ConnectorSyncInfo getRandomConnectorSyncInfo() {
 
     public static ConnectorFeatures getRandomConnectorFeatures() {
         return new ConnectorFeatures.Builder().setDocumentLevelSecurityEnabled(randomBoolean() ? randomConnectorFeatureEnabled() : null)
-            .setFilteringRules(randomFrom(new Boolean[] { null, randomBoolean() }))
-            .setFilteringAdvancedConfig(randomFrom(new Boolean[] { null, randomBoolean() }))
             .setIncrementalSyncEnabled(randomBoolean() ? randomConnectorFeatureEnabled() : null)
             .setNativeConnectorAPIKeysEnabled(randomBoolean() ? randomConnectorFeatureEnabled() : null)
             .setSyncRulesFeatures(randomBoolean() ? randomSyncRulesFeatures() : null)
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java
index 53efc2d0363c6..734c6eaf86965 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java
@@ -131,7 +131,6 @@ public void testToXContent() throws IOException {
                   "document_level_security":{
                      "enabled":true
                   },
-                  "filtering_advanced_config":true,
                   "sync_rules":{
                      "advanced":{
                         "enabled":false

From 968f07353f2ee4d2457a941dc67de0e151e37dee Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 28 May 2024 14:12:28 +0100
Subject: [PATCH 005/208] =?UTF-8?q?AwaitsFix=20for=C2=A0#109103?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../org/elasticsearch/cluster/coordination/NodeJoinTests.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
index b8dfdd8e91231..1b4ef86a85f7a 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
@@ -678,6 +678,7 @@ public void testBecomeFollowerFailsPendingJoin() throws Exception {
         assertFalse(isLocalNodeElectedMaster());
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/109103")
     public void testConcurrentJoining() {
         List<DiscoveryNode> masterNodes = IntStream.rangeClosed(1, randomIntBetween(2, 5))
             .mapToObj(nodeId -> newNode(nodeId, true))

From 6d864154cab1798fb300ebdaf0b949df1d2581e5 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Tue, 28 May 2024 15:21:33 +0200
Subject: [PATCH 006/208] [Inference API] Add Google AI Studio completion docs
 (#109089)

---
 .../inference/delete-inference.asciidoc       |  2 +-
 .../inference/get-inference.asciidoc          |  2 +-
 .../inference/inference-apis.asciidoc         |  2 +-
 .../inference/post-inference.asciidoc         |  2 +-
 .../inference/put-inference.asciidoc          | 52 ++++++++++++++++++-
 5 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/docs/reference/inference/delete-inference.asciidoc b/docs/reference/inference/delete-inference.asciidoc
index 4c7fb06d19ea6..89f76e6cef841 100644
--- a/docs/reference/inference/delete-inference.asciidoc
+++ b/docs/reference/inference/delete-inference.asciidoc
@@ -7,7 +7,7 @@ experimental[]
 Deletes an {infer} endpoint.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure or
+{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio or
 Hugging Face. For built-in models and models uploaded though Eland, the {infer}
 APIs offer an alternative way to use and manage trained models. However, if you
 do not plan to use the {infer} APIs to use these models or if you want to use
diff --git a/docs/reference/inference/get-inference.asciidoc b/docs/reference/inference/get-inference.asciidoc
index 71714c6254e64..339146adfece9 100644
--- a/docs/reference/inference/get-inference.asciidoc
+++ b/docs/reference/inference/get-inference.asciidoc
@@ -7,7 +7,7 @@ experimental[]
 Retrieves {infer} endpoint information.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure or
+{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio or
 Hugging Face. For built-in models and models uploaded though Eland, the {infer}
 APIs offer an alternative way to use and manage trained models. However, if you
 do not plan to use the {infer} APIs to use these models or if you want to use
diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc
index bd9f6308bfce6..539bba3f0d61f 100644
--- a/docs/reference/inference/inference-apis.asciidoc
+++ b/docs/reference/inference/inference-apis.asciidoc
@@ -5,7 +5,7 @@
 experimental[]
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure or
+{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio or
 Hugging Face. For built-in models and models uploaded though Eland, the {infer}
 APIs offer an alternative way to use and manage trained models. However, if you
 do not plan to use the {infer} APIs to use these models or if you want to use
diff --git a/docs/reference/inference/post-inference.asciidoc b/docs/reference/inference/post-inference.asciidoc
index b262eed3501bb..2b9e990140658 100644
--- a/docs/reference/inference/post-inference.asciidoc
+++ b/docs/reference/inference/post-inference.asciidoc
@@ -7,7 +7,7 @@ experimental[]
 Performs an inference task on an input text by using an {infer} endpoint.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure or
+{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio or
 Hugging Face. For built-in models and models uploaded though Eland, the {infer}
 APIs offer an alternative way to use and manage trained models. However, if you
 do not plan to use the {infer} APIs to use these models or if you want to use
diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc
index bb91e55ba99b9..cac03afd29562 100644
--- a/docs/reference/inference/put-inference.asciidoc
+++ b/docs/reference/inference/put-inference.asciidoc
@@ -8,7 +8,7 @@ Creates an {infer} endpoint to perform an {infer} task.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
 {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure
-OpenAI or Hugging Face. For built-in models and models uploaded though
+OpenAI, Google AI Studio or Hugging Face. For built-in models and models uploaded though
 Eland, the {infer} APIs offer an alternative way to use and manage trained
 models. However, if you do not plan to use the {infer} APIs to use these models
 or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
@@ -45,6 +45,7 @@ The following services are available through the {infer} API:
 * Azure OpenAI
 * Azure AI Studio
 * Elasticsearch (for built-in models and models uploaded through Eland)
+* Google AI Studio
 
 
 [discrete]
@@ -84,6 +85,7 @@ OpenAI service.
 * `azureaistudio`: specify the `completion` or `text_embedding` task type to use the Azure AI Studio service.
 * `elasticsearch`: specify the `text_embedding` task type to use the E5
 built-in model or text embedding models uploaded by Eland.
+* `googleaistudio`: specify the `completion` task to use the Google AI Studio service.
 
 `service_settings`::
 (Required, object)
@@ -282,6 +284,33 @@ To modify this, set the `requests_per_minute` setting of this object in your ser
 ```
 =====
 +
+.`service_settings` for the `googleiastudio` service
+[%collapsible%closed]
+=====
+`api_key`:::
+(Required, string)
+A valid API key for the Google Gemini API.
+
+`model_id`:::
+(Required, string)
+The name of the model to use for the {infer} task.
+You can find the supported models at https://ai.google.dev/gemini-api/docs/models/gemini[Gemini API models].
+
+`rate_limit`:::
+(Optional, object)
+By default, the `googleaistudio` service sets the number of requests allowed per minute to `360`.
+This helps to minimize the number of rate limit errors returned from Google AI Studio.
+To modify this, set the `requests_per_minute` setting of this object in your service settings:
++
+--
+```
+"rate_limit": {
+    "requests_per_minute": <<number_of_requests>>
+}
+```
+--
+=====
++
 .`service_settings` for the `elasticsearch` service
 [%collapsible%closed]
 =====
@@ -304,7 +333,6 @@ exceed the number of available processors per node divided by the number of
 allocations. Must be a power of 2. Max allowed value is 32.
 =====
 
-
 `task_settings`::
 (Optional, object)
 Settings to configure the {infer} task. These settings are specific to the
@@ -701,3 +729,23 @@ PUT _inference/completion/azure_ai_studio_completion
 // TEST[skip:TBD]
 
 The list of chat completion models that you can choose from in your deployment can be found in the https://ai.azure.com/explore/models?selectedTask=chat-completion[Azure AI Studio model explorer].
+
+[discrete]
+[[inference-example-googleaistudio]]
+===== Google AI Studio service
+
+The following example shows how to create an {infer} endpoint called
+`google_ai_studio_completion` to perform a `completion` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/completion/google_ai_studio_completion
+{
+    "service": "googleaistudio",
+    "service_settings": {
+        "api_key": "<api_key>>",
+        "model_id": "<model_id>"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]

From a80b7f62dbc99b3b6a5dc2b8d937ebd9fffbcdcd Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 28 May 2024 15:13:25 +0100
Subject: [PATCH 007/208] Fix trappy timeouts in `AcknowledgedRequest.Plain`
 (#109102)

Relates #107984
---
 .../api/_internal.delete_desired_nodes.json   | 10 +++++++++
 .../TransportDesiredNodesActionsIT.java       | 10 +++++++--
 .../support/master/AcknowledgedRequest.java   |  4 ++--
 .../cluster/RestDeleteDesiredNodesAction.java |  4 ++--
 .../license/DeleteLicenseRequestBuilder.java  | 22 -------------------
 .../license/LicensingClient.java              |  4 ----
 .../license/RestDeleteLicenseAction.java      |  5 +----
 ...lusterStateLicenseServiceClusterTests.java |  8 ++++++-
 .../license/LicensesTransportTests.java       |  7 ++++--
 .../IndexLifecycleInitialisationTests.java    | 10 +++++++--
 .../xpack/ilm/action/RestGetStatusAction.java |  4 +---
 .../ilm/LifecycleOperationSnapshotTests.java  |  8 +++++--
 .../lucene/bwc/ArchiveLicenseIntegTests.java  | 10 +++++++--
 .../SearchableSnapshotsLicenseIntegTests.java |  5 ++++-
 .../slm/action/RestGetSLMStatusAction.java    |  4 +---
 .../RestGetSnapshotLifecycleStatsAction.java  |  5 +----
 16 files changed, 64 insertions(+), 56 deletions(-)
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequestBuilder.java

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.delete_desired_nodes.json b/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.delete_desired_nodes.json
index ff53ad1d9f4cd..8b4759719a41b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.delete_desired_nodes.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.delete_desired_nodes.json
@@ -18,6 +18,16 @@
           ]
         }
       ]
+    },
+    "params":{
+      "master_timeout": {
+        "type": "time",
+        "description": "Timeout for connection to master node"
+      },
+      "timeout": {
+        "type": "time",
+        "description": "Timeout for acknowledgements from all nodes"
+      }
     }
   }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDesiredNodesActionsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDesiredNodesActionsIT.java
index 38fe1f8f918f8..63801f8c1e511 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDesiredNodesActionsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDesiredNodesActionsIT.java
@@ -249,7 +249,12 @@ public void testDeleteDesiredNodesTasksAreBatchedCorrectly() throws Exception {
 
         final List<ActionFuture<ActionResponse.Empty>> deleteDesiredNodesFutures = new ArrayList<>(15);
         for (int i = 0; i < 15; i++) {
-            deleteDesiredNodesFutures.add(client().execute(TransportDeleteDesiredNodesAction.TYPE, new AcknowledgedRequest.Plain()));
+            deleteDesiredNodesFutures.add(
+                client().execute(
+                    TransportDeleteDesiredNodesAction.TYPE,
+                    new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+                )
+            );
         }
 
         for (ActionFuture<ActionResponse.Empty> future : deleteDesiredNodesFutures) {
@@ -321,7 +326,8 @@ private UpdateDesiredNodesRequest randomDryRunUpdateDesiredNodesRequest(Settings
     }
 
     private void deleteDesiredNodes() {
-        client().execute(TransportDeleteDesiredNodesAction.TYPE, new AcknowledgedRequest.Plain()).actionGet();
+        client().execute(TransportDeleteDesiredNodesAction.TYPE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
+            .actionGet();
     }
 
     private DesiredNodes getLatestDesiredNodes() {
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
index b6389d0b112b6..a55467fbfadf8 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
@@ -98,8 +98,8 @@ public Plain(StreamInput in) throws IOException {
             super(in);
         }
 
-        public Plain() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Plain(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+            super(masterNodeTimeout, ackTimeout);
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteDesiredNodesAction.java
index 18045828f4401..ed20f63ec406e 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteDesiredNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteDesiredNodesAction.java
@@ -18,6 +18,7 @@
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 public class RestDeleteDesiredNodesAction extends BaseRestHandler {
@@ -33,8 +34,7 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        final AcknowledgedRequest.Plain deleteDesiredNodesRequest = new AcknowledgedRequest.Plain();
-        deleteDesiredNodesRequest.masterNodeTimeout(getMasterNodeTimeout(request));
+        final var deleteDesiredNodesRequest = new AcknowledgedRequest.Plain(getMasterNodeTimeout(request), getAckTimeout(request));
         return restChannel -> client.execute(
             TransportDeleteDesiredNodesAction.TYPE,
             deleteDesiredNodesRequest,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequestBuilder.java
deleted file mode 100644
index 3b581ec123a71..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseRequestBuilder.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.support.master.AcknowledgedRequest;
-import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-
-public class DeleteLicenseRequestBuilder extends AcknowledgedRequestBuilder<
-    AcknowledgedRequest.Plain,
-    AcknowledgedResponse,
-    DeleteLicenseRequestBuilder> {
-
-    public DeleteLicenseRequestBuilder(ElasticsearchClient client) {
-        super(client, TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain());
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
index a97643a54308b..5396d45126bb5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
@@ -24,10 +24,6 @@ public GetLicenseRequestBuilder prepareGetLicense() {
         return new GetLicenseRequestBuilder(client);
     }
 
-    public DeleteLicenseRequestBuilder prepareDeleteLicense() {
-        return new DeleteLicenseRequestBuilder(client);
-    }
-
     public PostStartTrialRequestBuilder preparePostStartTrial() {
         return new PostStartTrialRequestBuilder(client);
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
index 814911407377a..24c081c237ee0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
@@ -37,10 +37,7 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        AcknowledgedRequest.Plain deleteLicenseRequest = new AcknowledgedRequest.Plain();
-        deleteLicenseRequest.ackTimeout(getAckTimeout(request));
-        deleteLicenseRequest.masterNodeTimeout(getMasterNodeTimeout(request));
-
+        final var deleteLicenseRequest = new AcknowledgedRequest.Plain(getMasterNodeTimeout(request), getAckTimeout(request));
         return channel -> client.admin()
             .cluster()
             .execute(TransportDeleteLicenseAction.TYPE, deleteLicenseRequest, new RestToXContentListener<>(channel));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceClusterTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceClusterTests.java
index 5890434f88e6f..338c1ebfb4e21 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceClusterTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceClusterTests.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.license;
 
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
@@ -15,6 +16,7 @@
 
 import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
 import static org.elasticsearch.test.NodeRoles.addRoles;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 @ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0)
@@ -51,7 +53,11 @@ public void testClusterRestartWithLicense() throws Exception {
         logger.info("--> get and check signed license");
         assertThat(licensingClient.prepareGetLicense().get().license(), equalTo(license));
         logger.info("--> remove licenses");
-        licensingClient.prepareDeleteLicense().get();
+
+        assertAcked(
+            client().execute(TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
+                .get()
+        );
         assertOperationMode(License.OperationMode.BASIC);
 
         logger.info("--> restart all nodes");
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
index f2293ac6bd9a1..7e9152c0cf18e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
@@ -181,8 +182,10 @@ public void testRemoveLicensesSimple() throws Exception {
         GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
         assertThat(getLicenseResponse.license(), equalTo(goldLicense));
         // delete all licenses
-        DeleteLicenseRequestBuilder deleteLicenseRequestBuilder = new DeleteLicenseRequestBuilder(clusterAdmin());
-        AcknowledgedResponse deleteLicenseResponse = deleteLicenseRequestBuilder.get();
+        AcknowledgedResponse deleteLicenseResponse = clusterAdmin().execute(
+            TransportDeleteLicenseAction.TYPE,
+            new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+        ).get();
         assertThat(deleteLicenseResponse.isAcknowledged(), equalTo(true));
         // get licenses (expected no licenses)
         getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
index 404d9a05396e9..209330a3be2de 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
@@ -432,7 +432,10 @@ public void testCreatePolicyWhenStopped() throws Exception {
 
         assertAcked(client().execute(ILMActions.STOP, new StopILMRequest()).get());
         assertBusy(() -> {
-            OperationMode mode = client().execute(GetStatusAction.INSTANCE, new AcknowledgedRequest.Plain()).get().getMode();
+            OperationMode mode = client().execute(
+                GetStatusAction.INSTANCE,
+                new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+            ).get().getMode();
             logger.info("--> waiting for STOPPED, currently: {}", mode);
             assertThat(mode, equalTo(OperationMode.STOPPED));
         });
@@ -456,7 +459,10 @@ public void testCreatePolicyWhenStopped() throws Exception {
             is(both(greaterThanOrEqualTo(lowerBoundModifiedDate)).and(lessThanOrEqualTo(upperBoundModifiedDate)))
         );
         // assert ILM is still stopped
-        GetStatusAction.Response statusResponse = client().execute(GetStatusAction.INSTANCE, new AcknowledgedRequest.Plain()).get();
+        GetStatusAction.Response statusResponse = client().execute(
+            GetStatusAction.INSTANCE,
+            new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+        ).get();
         assertThat(statusResponse.getMode(), equalTo(OperationMode.STOPPED));
     }
 
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetStatusAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetStatusAction.java
index 5668b3241e6a2..3940b0a5e8ef8 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetStatusAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetStatusAction.java
@@ -34,9 +34,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        AcknowledgedRequest.Plain request = new AcknowledgedRequest.Plain();
-        request.ackTimeout(getAckTimeout(restRequest));
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        final var request = new AcknowledgedRequest.Plain(getMasterNodeTimeout(restRequest), getAckTimeout(restRequest));
         return channel -> client.execute(GetStatusAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
index fe0c905b35dad..22eef3d8940a3 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
@@ -125,10 +125,14 @@ public void testModeSnapshotRestore() throws Exception {
     }
 
     private OperationMode ilmMode() throws Exception {
-        return client().execute(GetStatusAction.INSTANCE, new AcknowledgedRequest.Plain()).get().getMode();
+        return client().execute(GetStatusAction.INSTANCE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
+            .get()
+            .getMode();
     }
 
     private OperationMode slmMode() throws Exception {
-        return client().execute(GetSLMStatusAction.INSTANCE, new AcknowledgedRequest.Plain()).get().getOperationMode();
+        return client().execute(GetSLMStatusAction.INSTANCE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
+            .get()
+            .getOperationMode();
     }
 }
diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java
index c7f00063161e1..0489390a21352 100644
--- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java
+++ b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java
@@ -57,7 +57,10 @@ public void testFeatureUsage() throws Exception {
     }
 
     public void testFailRestoreOnInvalidLicense() throws Exception {
-        assertAcked(client().execute(TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain()).get());
+        assertAcked(
+            client().execute(TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
+                .get()
+        );
         assertAcked(client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest()).get());
 
         ensureClusterSizeConsistency();
@@ -93,7 +96,10 @@ public void testShardAllocationOnInvalidLicense() throws Exception {
         assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
         ensureGreen(indexName);
 
-        assertAcked(client().execute(TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain()).get());
+        assertAcked(
+            client().execute(TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
+                .get()
+        );
         assertAcked(client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest()).get());
 
         ensureClusterSizeConsistency();
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java
index d5fcf0853cdae..3a218e4ca5e97 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java
@@ -78,7 +78,10 @@ public void createAndMountSearchableSnapshot() throws Exception {
         assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
         ensureGreen(indexName);
 
-        assertAcked(client().execute(TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain()).get());
+        assertAcked(
+            client().execute(TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
+                .get()
+        );
         assertAcked(client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest()).get());
 
         ensureClusterSizeConsistency();
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSLMStatusAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSLMStatusAction.java
index 51b8b5d934b86..7065af27a6f3a 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSLMStatusAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSLMStatusAction.java
@@ -37,9 +37,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        AcknowledgedRequest.Plain request = new AcknowledgedRequest.Plain();
-        request.ackTimeout(getAckTimeout(restRequest));
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        final var request = new AcknowledgedRequest.Plain(getMasterNodeTimeout(restRequest), getAckTimeout(restRequest));
         return channel -> client.execute(GetSLMStatusAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java
index c403c50715e73..a1a3f8fc7bc6b 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java
@@ -37,10 +37,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        AcknowledgedRequest.Plain req = new AcknowledgedRequest.Plain();
-        req.ackTimeout(getAckTimeout(request));
-        req.masterNodeTimeout(getMasterNodeTimeout(request));
-
+        final var req = new AcknowledgedRequest.Plain(getMasterNodeTimeout(request), getAckTimeout(request));
         return channel -> client.execute(GetSnapshotLifecycleStatsAction.INSTANCE, req, new RestToXContentListener<>(channel));
     }
 }

From 92dc76ee22637747ec8577a89f57f11bc302b9ec Mon Sep 17 00:00:00 2001
From: eyalkoren <41850454+eyalkoren@users.noreply.github.com>
Date: Tue, 28 May 2024 17:24:09 +0300
Subject: [PATCH 008/208] Raw mapping merge fix for properties field (#108867)

---
 docs/changelog/108867.yaml                    |  6 ++
 .../common/xcontent/XContentHelper.java       |  2 +-
 .../index/mapper/MapperService.java           |  7 +-
 .../index/mapper/MapperServiceTests.java      | 99 ++++++++++++++++++-
 4 files changed, 110 insertions(+), 4 deletions(-)
 create mode 100644 docs/changelog/108867.yaml

diff --git a/docs/changelog/108867.yaml b/docs/changelog/108867.yaml
new file mode 100644
index 0000000000000..545349dd84aeb
--- /dev/null
+++ b/docs/changelog/108867.yaml
@@ -0,0 +1,6 @@
+pr: 108867
+summary: Fix for raw mapping merge of fields named "properties"
+area: Mapping
+type: bug
+issues:
+ - 108866
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
index 2227c54871352..28df2fad32cbb 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
@@ -415,7 +415,7 @@ public static void mergeDefaults(Map<String, Object> content, Map<String, Object
      * Merges the map provided as the second parameter into the content of the first. Only does recursive merge for inner maps.
      * If a non-null {@link CustomMerge} is provided, it is applied whenever a merge is required, meaning - whenever both the first and
      * the second map has values for the same key. Otherwise, values from the first map will always have precedence, meaning - if the
-     * first map contains a key, its value will not be overriden.
+     * first map contains a key, its value will not be overridden.
      * @param first the map which serves as the merge base
      * @param second the map of which contents are merged into the base map
      * @param customMerge a custom merge rule to apply whenever a key has concrete values (i.e. not a map or a collection) in both maps
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 19f8da0954c5d..d3665c3b978bd 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -480,8 +480,11 @@ public Object merge(String parent, String key, Object oldValue, Object newValue)
                         if (baseMap.containsKey("subobjects")) {
                             mergedMappings.put("subobjects", baseMap.get("subobjects"));
                         }
-                        // recursively merge these two field mappings
-                        XContentHelper.merge(key, mergedMappings, mapToMerge, INSTANCE);
+                        // Recursively merge these two field mappings.
+                        // Since "key" is an arbitrary field name, for which we only need plain mapping subtrees merge, no need to pass it
+                        // to the recursion as it shouldn't affect the merge logic. Specifically, passing a parent may cause merge
+                        // failures of fields named "properties". See https://github.com/elastic/elasticsearch/issues/108866
+                        XContentHelper.merge(mergedMappings, mapToMerge, INSTANCE);
                         return mergedMappings;
                     } else {
                         // non-mergeable types - replace the entire mapping subtree for this field
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
index 0a49907b25567..bb5d50267642f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
@@ -1147,7 +1147,7 @@ public void testMultipleTypeMerges() throws IOException {
             }""", Strings.toString(mapperService.documentMapper().mapping(), true, true));
     }
 
-    public void testPropertiesField() throws IOException {
+    public void testPropertiesFieldSingleChildMerge() throws IOException {
         CompressedXContent mapping1 = new CompressedXContent("""
             {
               "properties": {
@@ -1238,6 +1238,103 @@ public void testPropertiesField() throws IOException {
         assertEquals("keyword", grandchildMapper.typeName());
     }
 
+    public void testPropertiesFieldMultiChildMerge() throws IOException {
+        CompressedXContent mapping1 = new CompressedXContent("""
+            {
+              "properties": {
+                "properties": {
+                  "properties": {
+                    "child1": {
+                      "type": "text",
+                      "fields": {
+                        "keyword": {
+                          "type": "keyword"
+                        }
+                      }
+                    },
+                    "child2": {
+                      "type": "text"
+                    },
+                    "child3": {
+                      "properties": {
+                        "grandchild": {
+                          "type": "text"
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }""");
+
+        CompressedXContent mapping2 = new CompressedXContent("""
+            {
+              "properties": {
+                "properties": {
+                  "properties": {
+                    "child2": {
+                      "type": "integer"
+                    },
+                    "child3": {
+                      "properties": {
+                        "grandchild": {
+                          "type": "long"
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }""");
+
+        MapperService mapperService = createMapperService(mapping(b -> {}));
+        mapperService.merge("_doc", List.of(mapping1, mapping2), MergeReason.INDEX_TEMPLATE);
+        assertEquals("""
+            {
+              "_doc" : {
+                "properties" : {
+                  "properties" : {
+                    "properties" : {
+                      "child1" : {
+                        "type" : "text",
+                        "fields" : {
+                          "keyword" : {
+                            "type" : "keyword"
+                          }
+                        }
+                      },
+                      "child2" : {
+                        "type" : "integer"
+                      },
+                      "child3" : {
+                        "properties" : {
+                          "grandchild" : {
+                            "type" : "long"
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }""", Strings.toString(mapperService.documentMapper().mapping(), true, true));
+
+        Mapper propertiesMapper = mapperService.documentMapper().mapping().getRoot().getMapper("properties");
+        assertThat(propertiesMapper, instanceOf(ObjectMapper.class));
+        Mapper childMapper = ((ObjectMapper) propertiesMapper).getMapper("child1");
+        assertThat(childMapper, instanceOf(FieldMapper.class));
+        assertEquals("text", childMapper.typeName());
+        assertEquals(2, childMapper.getTotalFieldsCount());
+        childMapper = ((ObjectMapper) propertiesMapper).getMapper("child2");
+        assertThat(childMapper, instanceOf(FieldMapper.class));
+        assertEquals("integer", childMapper.typeName());
+        assertEquals(1, childMapper.getTotalFieldsCount());
+        childMapper = ((ObjectMapper) propertiesMapper).getMapper("child3");
+        assertThat(childMapper, instanceOf(ObjectMapper.class));
+        Mapper grandchildMapper = ((ObjectMapper) childMapper).getMapper("grandchild");
+        assertEquals("long", grandchildMapper.typeName());
+    }
+
     public void testMergeUntilLimit() throws IOException {
         CompressedXContent mapping1 = new CompressedXContent("""
             {

From 0b7114f61cc1d93652de5f11c32062f7efba8d47 Mon Sep 17 00:00:00 2001
From: Henning Andersen <33268011+henningandersen@users.noreply.github.com>
Date: Tue, 28 May 2024 17:22:39 +0200
Subject: [PATCH 009/208] NodeJoinTests fix for future complete assertion
 (#109107)

NodeJoinTests.testConcurrentJoining use their own threads, but the thread name
parsing done for validating that wait and complete of future is on different
executors would pick up more or less random fields. Now disregard these test
threads for that assertion.

Closes #109103
---
 .../elasticsearch/common/util/concurrent/EsExecutors.java    | 5 ++++-
 .../elasticsearch/cluster/coordination/NodeJoinTests.java    | 5 ++---
 .../common/util/concurrent/EsExecutorsTests.java             | 4 ++++
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
index 9bf381e6f4719..04a71b2421ddc 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
@@ -270,7 +270,10 @@ public static String executorName(String threadName) {
         // subtract 2 to avoid the `]` of the thread number part.
         int executorNameEnd = threadName.lastIndexOf(']', threadName.length() - 2);
         int executorNameStart = threadName.lastIndexOf('[', executorNameEnd);
-        if (executorNameStart == -1 || executorNameEnd - executorNameStart <= 1) {
+        if (executorNameStart == -1
+            || executorNameEnd - executorNameStart <= 1
+            || threadName.startsWith("TEST-")
+            || threadName.startsWith("LuceneTestCase")) {
             return null;
         }
         return threadName.substring(executorNameStart + 1, executorNameEnd);
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
index 1b4ef86a85f7a..8a0f06d38fc43 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
@@ -678,7 +678,6 @@ public void testBecomeFollowerFailsPendingJoin() throws Exception {
         assertFalse(isLocalNodeElectedMaster());
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/109103")
     public void testConcurrentJoining() {
         List<DiscoveryNode> masterNodes = IntStream.rangeClosed(1, randomIntBetween(2, 5))
             .mapToObj(nodeId -> newNode(nodeId, true))
@@ -775,14 +774,14 @@ public void testConcurrentJoining() {
         final List<Thread> joinThreads = Stream.concat(correctJoinRequests.stream().map(joinRequest -> new Thread(() -> {
             safeAwait(barrier);
             joinNode(joinRequest);
-        }, "process " + joinRequest)), possiblyFailingJoinRequests.stream().map(joinRequest -> new Thread(() -> {
+        }, "TEST-process " + joinRequest)), possiblyFailingJoinRequests.stream().map(joinRequest -> new Thread(() -> {
             safeAwait(barrier);
             try {
                 joinNode(joinRequest);
             } catch (CoordinationStateRejectedException e) {
                 // ignore - these requests are expected to fail
             }
-        }, "process " + joinRequest))).toList();
+        }, "TEST-process " + joinRequest))).toList();
 
         assertionThread.start();
         joinThreads.forEach(Thread::start);
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java
index df7b02c2309a3..e267b6c3b737d 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java
@@ -37,6 +37,7 @@
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  * Tests for EsExecutors and its components like EsAbortPolicy.
@@ -646,6 +647,9 @@ public void testParseExecutorName() throws InterruptedException {
         final var thread = threadFactory.newThread(() -> {});
         try {
             assertThat(EsExecutors.executorName(thread.getName()), equalTo(executorName));
+            assertThat(EsExecutors.executorName(thread), equalTo(executorName));
+            assertThat(EsExecutors.executorName("TEST-" + thread.getName()), is(nullValue()));
+            assertThat(EsExecutors.executorName("LuceneTestCase" + thread.getName()), is(nullValue()));
         } finally {
             thread.join();
         }

From efd450ee19d18e26d4059671c6fbc8de356bf6a7 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 28 May 2024 16:43:54 +0100
Subject: [PATCH 010/208] Clarify that red/yellow health must be addressed
 (#109090)

We don't expect a cluster to run with `yellow` health for an extended
period of time, but it's not clear from these docs that it's important
to bring the cluster back to `green` health ASAP. This commit clarifies
these docs.
---
 .../red-yellow-cluster-status.asciidoc        | 25 +++++++++++++++----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc b/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc
index 4b49e191f4e16..cae4eb99dd54a 100644
--- a/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc
@@ -1,9 +1,24 @@
 [[red-yellow-cluster-status]]
-=== Red or yellow cluster status
-
-A red or yellow cluster status indicates one or more shards are missing or
-unallocated. These unassigned shards increase your risk of data loss and can
-degrade cluster performance.
+=== Red or yellow cluster health status
+
+A red or yellow cluster health status indicates one or more shards are not assigned to
+a node. 
+
+* **Red health status**: The cluster has some unassigned primary shards, which
+means that some operations such as searches and indexing may fail. 
+* **Yellow health status**: The cluster has no unassigned primary shards but some 
+unassigned replica shards. This increases your risk of data loss and can degrade 
+cluster performance.
+
+When your cluster has a red or yellow health status, it will continue to process
+searches and indexing where possible, but may delay certain management and
+cleanup activities until the cluster returns to green health status. For instance,
+some <<index-lifecycle-management,{ilm-init}>> actions require the index on which they
+operate to have a green health status.
+
+In many cases, your cluster will recover to green health status automatically. 
+If the cluster doesn't automatically recover, then you must <<fix-red-yellow-cluster-status,manually address>> 
+the remaining problems so management and cleanup activities can proceed.
 
 [discrete]
 [[diagnose-cluster-status]]

From 398c22c06c08686a7c1619ff7a353f73d036e9e9 Mon Sep 17 00:00:00 2001
From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com>
Date: Tue, 28 May 2024 11:59:19 -0400
Subject: [PATCH 011/208] Revert "[ML] Refactor the Embedding classes
 (#107516)" (#109114)

* Revert "[ML] Refactor the Embedding classes (#107516)"

This reverts commit 843579652c4f1b1842ef44ffbb496995f00b5b71.
---
 .../core/inference/results/ByteEmbedding.java |  98 ----------
 .../ChunkedSparseEmbeddingResults.java        |  18 +-
 .../ChunkedTextEmbeddingByteResults.java      |  80 +++++++-
 .../ChunkedTextEmbeddingFloatResults.java     |  76 +++++++-
 .../results/ChunkedTextEmbeddingResults.java  |   2 +-
 .../core/inference/results/Embedding.java     |  72 -------
 .../inference/results/EmbeddingChunk.java     |  79 --------
 .../core/inference/results/EmbeddingInt.java  |  12 ++
 .../inference/results/EmbeddingResults.java   |  39 ----
 .../inference/results/FloatEmbedding.java     |  98 ----------
 .../inference/results/SparseEmbedding.java    | 170 -----------------
 .../results/SparseEmbeddingResults.java       |  68 ++++++-
 .../core/inference/results/TextEmbedding.java |   2 +-
 .../results/TextEmbeddingByteResults.java     | 104 +++++++++--
 .../results/TextEmbeddingResults.java         | 101 ++++++++--
 .../inference/results/TextEmbeddingUtils.java |   2 +-
 .../MlInferenceNamedXContentProvider.java     |   4 -
 .../results/TextEmbeddingByteResults.java     | 100 ----------
 .../TextEmbeddingByteResultsTests.java        |  61 ------
 .../TestDenseInferenceServiceExtension.java   |   5 +-
 .../TestSparseInferenceServiceExtension.java  |   9 +-
 .../common/EmbeddingRequestChunker.java       | 141 ++------------
 .../CohereEmbeddingsResponseEntity.java       |  13 +-
 .../HuggingFaceElserResponseEntity.java       |  14 +-
 .../HuggingFaceEmbeddingsResponseEntity.java  |  10 +-
 .../OpenAiEmbeddingsResponseEntity.java       |   7 +-
 .../common/EmbeddingRequestChunkerTests.java  | 175 +++---------------
 ...AiStudioEmbeddingsResponseEntityTests.java |   3 +-
 .../CohereEmbeddingsResponseEntityTests.java  |  38 +++-
 ...gingFaceEmbeddingsResponseEntityTests.java |  33 +++-
 .../OpenAiEmbeddingsResponseEntityTests.java  |  23 ++-
 .../rest/RestInferenceActionTests.java        |   5 +-
 .../ChunkedSparseEmbeddingResultsTests.java   |   5 +-
 .../ChunkedTextEmbeddingByteResultsTests.java |  21 ++-
 ...ChunkedTextEmbeddingFloatResultsTests.java |   8 +-
 .../ChunkedTextEmbeddingResultsTests.java     |   5 +-
 .../results/SparseEmbeddingResultsTests.java  |  46 ++---
 .../TextEmbeddingByteResultsTests.java        |  32 ++--
 .../results/TextEmbeddingResultsTests.java    |  32 ++--
 .../inference/services/ServiceUtilsTests.java |   4 +-
 .../services/cohere/CohereServiceTests.java   |   6 +-
 .../services/openai/OpenAiServiceTests.java   |   6 +-
 42 files changed, 614 insertions(+), 1213 deletions(-)
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ByteEmbedding.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/Embedding.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingChunk.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingInt.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingResults.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/FloatEmbedding.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbedding.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingByteResults.java
 delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingByteResultsTests.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ByteEmbedding.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ByteEmbedding.java
deleted file mode 100644
index 9fb017b75abb5..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ByteEmbedding.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.inference.results;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-public class ByteEmbedding extends Embedding<ByteEmbedding.ByteArrayWrapper> {
-
-    public static ByteEmbedding of(List<Byte> embedding) {
-        byte[] embeddingBytes = new byte[embedding.size()];
-        for (int i = 0; i < embedding.size(); i++) {
-            embeddingBytes[i] = embedding.get(i);
-        }
-        return new ByteEmbedding(embeddingBytes);
-    }
-
-    /**
-     * Wrapper so around a primitive byte array so that it can be
-     * treated as a generic
-     */
-    public static class ByteArrayWrapper implements EmbeddingValues {
-
-        final byte[] bytes;
-
-        public ByteArrayWrapper(byte[] bytes) {
-            this.bytes = bytes;
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return valuesToXContent(EMBEDDING, builder, params);
-        }
-
-        @Override
-        public int size() {
-            return bytes.length;
-        }
-
-        @Override
-        public XContentBuilder valuesToXContent(String fieldName, XContentBuilder builder, Params params) throws IOException {
-            builder.startArray(fieldName);
-            for (var value : bytes) {
-                builder.value(value);
-            }
-            builder.endArray();
-            return builder;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            ByteArrayWrapper that = (ByteArrayWrapper) o;
-            return Arrays.equals(bytes, that.bytes);
-        }
-
-        @Override
-        public int hashCode() {
-            return Arrays.hashCode(bytes);
-        }
-    }
-
-    public ByteEmbedding(StreamInput in) throws IOException {
-        this(in.readByteArray());
-    }
-
-    public ByteEmbedding(byte[] embedding) {
-        super(new ByteArrayWrapper(embedding));
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeByteArray(embedding.bytes);
-    }
-
-    public byte[] bytes() {
-        return embedding.bytes;
-    }
-
-    public float[] toFloatArray() {
-        float[] floatArray = new float[embedding.bytes.length];
-        for (int i = 0; i < embedding.bytes.length; i++) {
-            floatArray[i] = ((Byte) embedding.bytes[i]).floatValue();
-        }
-        return floatArray;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java
index 5b57a8da9d37c..c91d0dc6fd538 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedSparseEmbeddingResults.java
@@ -49,7 +49,7 @@ public static List<ChunkedInferenceServiceResults> of(List<String> inputs, Spars
         return results;
     }
 
-    public static ChunkedSparseEmbeddingResults of(String input, SparseEmbedding embedding) {
+    public static ChunkedSparseEmbeddingResults of(String input, SparseEmbeddingResults.Embedding embedding) {
         var weightedTokens = embedding.tokens()
             .stream()
             .map(weightedToken -> new WeightedToken(weightedToken.token(), weightedToken.weight()))
@@ -58,22 +58,6 @@ public static ChunkedSparseEmbeddingResults of(String input, SparseEmbedding emb
         return new ChunkedSparseEmbeddingResults(List.of(new ChunkedTextExpansionResults.ChunkedResult(input, weightedTokens)));
     }
 
-    public static ChunkedSparseEmbeddingResults of(List<EmbeddingChunk<SparseEmbedding.WeightedTokens>> embeddingChunks) {
-        var ch = embeddingChunks.stream()
-            .map(
-                chunk -> new ChunkedTextExpansionResults.ChunkedResult(
-                    chunk.matchedText(),
-                    chunk.embedding().embedding.tokens()
-                        .stream()
-                        .map(weightedToken -> new WeightedToken(weightedToken.token(), weightedToken.weight()))
-                        .toList()
-                )
-            )
-            .toList();
-
-        return new ChunkedSparseEmbeddingResults(ch);
-    }
-
     private final List<ChunkedTextExpansionResults.ChunkedResult> chunkedResults;
 
     public ChunkedSparseEmbeddingResults(List<ChunkedTextExpansionResults.ChunkedResult> chunks) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingByteResults.java
index b88b502a76195..86ea70ddd62dd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingByteResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingByteResults.java
@@ -7,22 +7,26 @@
 
 package org.elasticsearch.xpack.core.inference.results;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.InferenceResults;
+import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.xpack.core.inference.results.TextEmbeddingUtils.validateInputSizeAgainstEmbeddings;
 
-public record ChunkedTextEmbeddingByteResults(List<EmbeddingChunk<ByteEmbedding.ByteArrayWrapper>> chunks, boolean isTruncated)
-    implements
-        ChunkedInferenceServiceResults {
+public record ChunkedTextEmbeddingByteResults(List<EmbeddingChunk> chunks, boolean isTruncated) implements ChunkedInferenceServiceResults {
 
     public static final String NAME = "chunked_text_embedding_service_byte_results";
     public static final String FIELD_NAME = "text_embedding_byte_chunk";
@@ -37,22 +41,23 @@ public static List<ChunkedInferenceServiceResults> of(List<String> inputs, TextE
 
         var results = new ArrayList<ChunkedInferenceServiceResults>(inputs.size());
         for (int i = 0; i < inputs.size(); i++) {
-            results.add(of(inputs.get(i), textEmbeddings.embeddings().get(i).getEmbedding().bytes));
+            results.add(of(inputs.get(i), textEmbeddings.embeddings().get(i).values()));
         }
 
         return results;
     }
 
     public static ChunkedTextEmbeddingByteResults of(String input, byte[] byteEmbeddings) {
-        return new ChunkedTextEmbeddingByteResults(List.of(new EmbeddingChunk<>(input, new ByteEmbedding(byteEmbeddings))), false);
+        return new ChunkedTextEmbeddingByteResults(List.of(new EmbeddingChunk(input, byteEmbeddings)), false);
     }
 
     public ChunkedTextEmbeddingByteResults(StreamInput in) throws IOException {
-        this(in.readCollectionAsList(in1 -> new EmbeddingChunk<>(in1.readString(), new ByteEmbedding(in1))), in.readBoolean());
+        this(in.readCollectionAsList(EmbeddingChunk::new), in.readBoolean());
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        // TODO add isTruncated flag
         builder.startArray(FIELD_NAME);
         for (var embedding : chunks) {
             embedding.toXContent(builder, params);
@@ -87,7 +92,68 @@ public String getWriteableName() {
         return NAME;
     }
 
-    public List<EmbeddingChunk<ByteEmbedding.ByteArrayWrapper>> getChunks() {
+    public List<EmbeddingChunk> getChunks() {
         return chunks;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ChunkedTextEmbeddingByteResults that = (ChunkedTextEmbeddingByteResults) o;
+        return isTruncated == that.isTruncated && Objects.equals(chunks, that.chunks);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(chunks, isTruncated);
+    }
+
+    public record EmbeddingChunk(String matchedText, byte[] embedding) implements Writeable, ToXContentObject {
+
+        public EmbeddingChunk(StreamInput in) throws IOException {
+            this(in.readString(), in.readByteArray());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(matchedText);
+            out.writeByteArray(embedding);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(ChunkedNlpInferenceResults.TEXT, matchedText);
+
+            builder.startArray(ChunkedNlpInferenceResults.INFERENCE);
+            for (byte value : embedding) {
+                builder.value(value);
+            }
+            builder.endArray();
+
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            EmbeddingChunk that = (EmbeddingChunk) o;
+            return Objects.equals(matchedText, that.matchedText) && Arrays.equals(embedding, that.embedding);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(matchedText);
+            result = 31 * result + Arrays.hashCode(embedding);
+            return result;
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingFloatResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingFloatResults.java
index 65370277aba34..4fcd5a53fc287 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingFloatResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingFloatResults.java
@@ -7,30 +7,35 @@
 
 package org.elasticsearch.xpack.core.inference.results;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
-public record ChunkedTextEmbeddingFloatResults(List<EmbeddingChunk<FloatEmbedding.FloatArrayWrapper>> chunks)
-    implements
-        ChunkedInferenceServiceResults {
+public record ChunkedTextEmbeddingFloatResults(List<EmbeddingChunk> chunks) implements ChunkedInferenceServiceResults {
 
     public static final String NAME = "chunked_text_embedding_service_float_results";
     public static final String FIELD_NAME = "text_embedding_float_chunk";
 
     public ChunkedTextEmbeddingFloatResults(StreamInput in) throws IOException {
-        this(in.readCollectionAsList(in1 -> new EmbeddingChunk<>(in1.readString(), new FloatEmbedding(in1))));
+        this(in.readCollectionAsList(EmbeddingChunk::new));
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        // TODO add isTruncated flag
         builder.startArray(FIELD_NAME);
         for (var embedding : chunks) {
             embedding.toXContent(builder, params);
@@ -64,8 +69,69 @@ public String getWriteableName() {
         return NAME;
     }
 
-    public List<EmbeddingChunk<FloatEmbedding.FloatArrayWrapper>> getChunks() {
+    public List<EmbeddingChunk> getChunks() {
         return chunks;
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ChunkedTextEmbeddingFloatResults that = (ChunkedTextEmbeddingFloatResults) o;
+        return Objects.equals(chunks, that.chunks);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(chunks);
+    }
+
+    public record EmbeddingChunk(String matchedText, float[] embedding) implements Writeable, ToXContentObject {
+
+        public EmbeddingChunk(StreamInput in) throws IOException {
+            this(in.readString(), in.readFloatArray());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(matchedText);
+            out.writeFloatArray(embedding);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(ChunkedNlpInferenceResults.TEXT, matchedText);
+
+            builder.startArray(ChunkedNlpInferenceResults.INFERENCE);
+            for (float value : embedding) {
+                builder.value(value);
+            }
+            builder.endArray();
+
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            EmbeddingChunk that = (EmbeddingChunk) o;
+            return Objects.equals(matchedText, that.matchedText) && Arrays.equals(embedding, that.embedding);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(matchedText);
+            result = 31 * result + Arrays.hashCode(embedding);
+            return result;
+        }
+    }
+
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java
index 5553230cb7e9f..f09eafc1591dd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedTextEmbeddingResults.java
@@ -44,7 +44,7 @@ public static List<ChunkedInferenceServiceResults> of(List<String> inputs, TextE
 
         var results = new ArrayList<ChunkedInferenceServiceResults>(inputs.size());
         for (int i = 0; i < inputs.size(); i++) {
-            results.add(ChunkedTextEmbeddingResults.of(inputs.get(i), textEmbeddings.embeddings().get(i).getEmbedding().floats));
+            results.add(ChunkedTextEmbeddingResults.of(inputs.get(i), textEmbeddings.embeddings().get(i).values()));
         }
 
         return results;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/Embedding.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/Embedding.java
deleted file mode 100644
index 75eeb6ff01f73..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/Embedding.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.inference.results;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xcontent.ToXContentFragment;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
-public abstract class Embedding<T extends Embedding.EmbeddingValues> implements Writeable, ToXContentObject {
-    public static final String EMBEDDING = "embedding";
-
-    public interface EmbeddingValues extends ToXContentFragment {
-        int size();
-
-        XContentBuilder valuesToXContent(String fieldName, XContentBuilder builder, Params params) throws IOException;
-    }
-
-    protected final T embedding;
-
-    protected Embedding(T embedding) {
-        this.embedding = embedding;
-    }
-
-    public T getEmbedding() {
-        return embedding;
-    }
-
-    public Map<String, Object> asMap() {
-        return Map.of(EMBEDDING, embedding);
-    }
-
-    public int getSize() {
-        return embedding.size();
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        embedding.valuesToXContent(EMBEDDING, builder, params);
-        builder.endObject();
-        return builder;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        Embedding<?> embedding1 = (Embedding<?>) o;
-        return Objects.equals(embedding, embedding1.embedding);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(embedding);
-    }
-
-    @Override
-    public String toString() {
-        return Strings.toString(this);
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingChunk.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingChunk.java
deleted file mode 100644
index 9f0f3bc7fe952..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingChunk.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.inference.results;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-public class EmbeddingChunk<T extends Embedding.EmbeddingValues> implements Writeable, ToXContentObject {
-
-    private final String matchedText;
-    private final Embedding<T> embedding;
-
-    public EmbeddingChunk(String matchedText, Embedding<T> embedding) {
-        this.matchedText = matchedText;
-        this.embedding = embedding;
-    }
-
-    public String matchedText() {
-        return matchedText;
-    }
-
-    public Embedding<T> embedding() {
-        return embedding;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(matchedText);
-        embedding.writeTo(out);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.field(ChunkedNlpInferenceResults.TEXT, matchedText);
-        embedding.embedding.valuesToXContent(ChunkedNlpInferenceResults.INFERENCE, builder, params);
-        builder.endObject();
-        return builder;
-    }
-
-    public Map<String, Object> asMap() {
-        var map = new HashMap<String, Object>();
-        map.put(ChunkedNlpInferenceResults.TEXT, matchedText);
-        map.put(ChunkedNlpInferenceResults.INFERENCE, embedding.getEmbedding());
-        return map;
-    }
-
-    @Override
-    public String toString() {
-        return Strings.toString(this);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        EmbeddingChunk<?> that = (EmbeddingChunk<?>) o;
-        return Objects.equals(matchedText, that.matchedText) && Objects.equals(embedding, that.embedding);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(matchedText, embedding);
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingInt.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingInt.java
new file mode 100644
index 0000000000000..05fc8a3cef1b6
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingInt.java
@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.inference.results;
+
+public interface EmbeddingInt {
+    int getSize();
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingResults.java
deleted file mode 100644
index 851a320c262a5..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingResults.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.inference.results;
-
-import org.elasticsearch.inference.InferenceServiceResults;
-
-import java.util.List;
-
-public interface EmbeddingResults {
-    List<? extends Embedding<?>> embeddings();
-
-    EmbeddingType embeddingType();
-
-    enum EmbeddingType {
-        SPARSE {
-            public Class<? extends InferenceServiceResults> matchedClass() {
-                return SparseEmbeddingResults.class;
-            };
-        },
-        FLOAT {
-            public Class<? extends InferenceServiceResults> matchedClass() {
-                return TextEmbeddingResults.class;
-            };
-        },
-
-        BYTE {
-            public Class<? extends InferenceServiceResults> matchedClass() {
-                return TextEmbeddingByteResults.class;
-            };
-        };
-
-        public abstract Class<? extends InferenceServiceResults> matchedClass();
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/FloatEmbedding.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/FloatEmbedding.java
deleted file mode 100644
index 18bdff1a6f47d..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/FloatEmbedding.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.inference.results;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-public class FloatEmbedding extends Embedding<FloatEmbedding.FloatArrayWrapper> {
-
-    public static FloatEmbedding of(List<Float> embedding) {
-        float[] embeddingFloats = new float[embedding.size()];
-        for (int i = 0; i < embedding.size(); i++) {
-            embeddingFloats[i] = embedding.get(i);
-        }
-        return new FloatEmbedding(embeddingFloats);
-    }
-
-    public static class FloatArrayWrapper implements EmbeddingValues {
-
-        final float[] floats;
-
-        public FloatArrayWrapper(float[] floats) {
-            this.floats = floats;
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return valuesToXContent(EMBEDDING, builder, params);
-        }
-
-        @Override
-        public int size() {
-            return floats.length;
-        }
-
-        @Override
-        public XContentBuilder valuesToXContent(String fieldName, XContentBuilder builder, Params params) throws IOException {
-            builder.startArray(fieldName);
-            for (var value : floats) {
-                builder.value(value);
-            }
-            builder.endArray();
-            return builder;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            FloatArrayWrapper that = (FloatArrayWrapper) o;
-            return Arrays.equals(floats, that.floats);
-        }
-
-        @Override
-        public int hashCode() {
-            return Arrays.hashCode(floats);
-        }
-    }
-
-    public FloatEmbedding(StreamInput in) throws IOException {
-        this(in.readFloatArray());
-    }
-
-    public FloatEmbedding(float[] embedding) {
-        super(new FloatArrayWrapper(embedding));
-    }
-
-    public float[] asFloatArray() {
-        return embedding.floats;
-    }
-
-    public double[] asDoubleArray() {
-        var result = new double[embedding.floats.length];
-        for (int i = 0; i < embedding.floats.length; i++) {
-            result[i] = embedding.floats[i];
-        }
-        return result;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeFloatArray(embedding.floats);
-    }
-
-    public static FloatEmbedding of(org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults embeddingResult) {
-        return new FloatEmbedding(embeddingResult.getInferenceAsFloat());
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbedding.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbedding.java
deleted file mode 100644
index 18b93dd1fef8d..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbedding.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.inference.results;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xcontent.ToXContentFragment;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-public class SparseEmbedding extends Embedding<SparseEmbedding.WeightedTokens> {
-
-    public static final String IS_TRUNCATED = "is_truncated";
-
-    public static SparseEmbedding fromMlResults(
-        List<org.elasticsearch.xpack.core.ml.search.WeightedToken> weightedTokens,
-        boolean isTruncated
-    ) {
-        return new SparseEmbedding(
-            new WeightedTokens(weightedTokens.stream().map(token -> new WeightedToken(token.token(), token.weight())).toList()),
-            isTruncated
-        );
-    }
-
-    private final boolean isTruncated;
-
-    public SparseEmbedding(StreamInput in) throws IOException {
-        this(new WeightedTokens(in.readCollectionAsImmutableList(SparseEmbedding.WeightedToken::new)), in.readBoolean());
-    }
-
-    public SparseEmbedding(WeightedTokens embedding, boolean isTruncated) {
-        super(embedding);
-        this.isTruncated = isTruncated;
-    }
-
-    public SparseEmbedding(List<WeightedToken> tokens, boolean isTruncated) {
-        super(new WeightedTokens(tokens));
-        this.isTruncated = isTruncated;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeCollection(embedding.tokens);
-        out.writeBoolean(isTruncated);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.field(IS_TRUNCATED, isTruncated);
-        embedding.toXContent(builder, params);
-        builder.endObject();
-        return builder;
-    }
-
-    public boolean isTruncated() {
-        return isTruncated;
-    }
-
-    public Map<String, Object> asMap() {
-        var embeddingMap = new LinkedHashMap<String, Object>(
-            embedding.tokens.stream().collect(Collectors.toMap(WeightedToken::token, WeightedToken::weight))
-        );
-
-        return new LinkedHashMap<>(Map.of(IS_TRUNCATED, isTruncated, EMBEDDING, embeddingMap));
-    }
-
-    public List<WeightedToken> tokens() {
-        return embedding.tokens;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        if (super.equals(o) == false) return false;
-        SparseEmbedding that = (SparseEmbedding) o;
-        return isTruncated == that.isTruncated;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), isTruncated);
-    }
-
-    public static class WeightedTokens implements Embedding.EmbeddingValues {
-        private final List<WeightedToken> tokens;
-
-        public WeightedTokens(List<WeightedToken> tokens) {
-            this.tokens = tokens;
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return valuesToXContent(EMBEDDING, builder, params);
-        }
-
-        @Override
-        public int size() {
-            return tokens.size();
-        }
-
-        @Override
-        public XContentBuilder valuesToXContent(String fieldName, XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(fieldName);
-            for (var weightedToken : tokens) {
-                weightedToken.toXContent(builder, params);
-            }
-            builder.endObject();
-            return builder;
-        }
-
-        public List<WeightedToken> tokens() {
-            return tokens;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            WeightedTokens that = (WeightedTokens) o;
-            return Objects.equals(tokens, that.tokens);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(tokens);
-        }
-    }
-
-    public record WeightedToken(String token, float weight) implements Writeable, ToXContentFragment {
-        public WeightedToken(StreamInput in) throws IOException {
-            this(in.readString(), in.readFloat());
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(token);
-            out.writeFloat(weight);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(token, weight);
-            return builder;
-        }
-
-        public Map<String, Object> asMap() {
-            return Map.of(token, weight);
-        }
-
-        @Override
-        public String toString() {
-            return Strings.toString(this);
-        }
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
index f108f9e86416c..1db6dcc802d00 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java
@@ -8,12 +8,15 @@
 package org.elasticsearch.xpack.core.inference.results;
 
 import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults;
 import org.elasticsearch.xpack.core.ml.search.WeightedToken;
@@ -23,24 +26,25 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig.DEFAULT_RESULTS_FIELD;
 
-public record SparseEmbeddingResults(List<SparseEmbedding> embeddings) implements InferenceServiceResults, EmbeddingResults {
+public record SparseEmbeddingResults(List<Embedding> embeddings) implements InferenceServiceResults {
 
     public static final String NAME = "sparse_embedding_results";
     public static final String SPARSE_EMBEDDING = TaskType.SPARSE_EMBEDDING.toString();
 
     public SparseEmbeddingResults(StreamInput in) throws IOException {
-        this(in.readCollectionAsList(SparseEmbedding::new));
+        this(in.readCollectionAsList(Embedding::new));
     }
 
     public static SparseEmbeddingResults of(List<? extends InferenceResults> results) {
-        List<SparseEmbedding> embeddings = new ArrayList<>(results.size());
+        List<Embedding> embeddings = new ArrayList<>(results.size());
 
         for (InferenceResults result : results) {
             if (result instanceof TextExpansionResults expansionResults) {
-                embeddings.add(SparseEmbedding.fromMlResults(expansionResults.getWeightedTokens(), expansionResults.isTruncated()));
+                embeddings.add(Embedding.create(expansionResults.getWeightedTokens(), expansionResults.isTruncated()));
             } else if (result instanceof org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults errorResult) {
                 if (errorResult.getException() instanceof ElasticsearchStatusException statusException) {
                     throw statusException;
@@ -67,7 +71,7 @@ public static SparseEmbeddingResults of(List<? extends InferenceResults> results
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startArray(SPARSE_EMBEDDING);
 
-        for (var embedding : embeddings) {
+        for (Embedding embedding : embeddings) {
             embedding.toXContent(builder, params);
         }
 
@@ -108,14 +112,60 @@ public List<? extends InferenceResults> transformToLegacyFormat() {
                         .stream()
                         .map(weightedToken -> new WeightedToken(weightedToken.token(), weightedToken.weight()))
                         .toList(),
-                    embedding.isTruncated()
+                    embedding.isTruncated
                 )
             )
             .toList();
     }
 
-    @Override
-    public EmbeddingType embeddingType() {
-        return EmbeddingType.SPARSE;
+    public record Embedding(List<WeightedToken> tokens, boolean isTruncated) implements Writeable, ToXContentObject {
+
+        public static final String EMBEDDING = "embedding";
+        public static final String IS_TRUNCATED = "is_truncated";
+
+        public Embedding(StreamInput in) throws IOException {
+            this(in.readCollectionAsList(WeightedToken::new), in.readBoolean());
+        }
+
+        public static Embedding create(List<WeightedToken> weightedTokens, boolean isTruncated) {
+            return new Embedding(
+                weightedTokens.stream().map(token -> new WeightedToken(token.token(), token.weight())).toList(),
+                isTruncated
+            );
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeCollection(tokens);
+            out.writeBoolean(isTruncated);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(IS_TRUNCATED, isTruncated);
+            builder.startObject(EMBEDDING);
+
+            for (var weightedToken : tokens) {
+                weightedToken.toXContent(builder, params);
+            }
+
+            builder.endObject();
+            builder.endObject();
+            return builder;
+        }
+
+        public Map<String, Object> asMap() {
+            var embeddingMap = new LinkedHashMap<String, Object>(
+                tokens.stream().collect(Collectors.toMap(WeightedToken::token, WeightedToken::weight))
+            );
+
+            return new LinkedHashMap<>(Map.of(IS_TRUNCATED, isTruncated, EMBEDDING, embeddingMap));
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbedding.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbedding.java
index ef406f5c956ad..a185c2938223e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbedding.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbedding.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.inference.results;
 
-public interface TextEmbedding extends EmbeddingResults {
+public interface TextEmbedding {
 
     /**
      * Returns the first text embedding entry in the result list's array size.
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingByteResults.java
index 743795d0530f5..04986b2d957d7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingByteResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingByteResults.java
@@ -9,14 +9,18 @@
 
 package org.elasticsearch.xpack.core.inference.results;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -39,27 +43,12 @@
  *     ]
  * }
  */
-public class TextEmbeddingByteResults implements InferenceServiceResults, TextEmbedding {
+public record TextEmbeddingByteResults(List<Embedding> embeddings) implements InferenceServiceResults, TextEmbedding {
     public static final String NAME = "text_embedding_service_byte_results";
     public static final String TEXT_EMBEDDING_BYTES = "text_embedding_bytes";
 
-    private final List<ByteEmbedding> embeddings;
-
-    public TextEmbeddingByteResults(List<ByteEmbedding> embeddings) {
-        this.embeddings = embeddings;
-    }
-
     public TextEmbeddingByteResults(StreamInput in) throws IOException {
-        this(in.readCollectionAsList(ByteEmbedding::new));
-    }
-
-    public List<ByteEmbedding> embeddings() {
-        return embeddings;
-    }
-
-    @Override
-    public EmbeddingType embeddingType() {
-        return EmbeddingType.BYTE;
+        this(in.readCollectionAsList(Embedding::new));
     }
 
     @Override
@@ -70,7 +59,7 @@ public int getFirstEmbeddingSize() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startArray(TEXT_EMBEDDING_BYTES);
-        for (var embedding : embeddings) {
+        for (Embedding embedding : embeddings) {
             embedding.toXContent(builder, params);
         }
         builder.endArray();
@@ -91,9 +80,9 @@ public String getWriteableName() {
     public List<? extends InferenceResults> transformToCoordinationFormat() {
         return embeddings.stream()
             .map(
-                embedding -> new org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingByteResults(
+                embedding -> new org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults(
                     TEXT_EMBEDDING_BYTES,
-                    embedding.bytes(),
+                    embedding.toDoubleArray(),
                     false
                 )
             )
@@ -125,7 +114,82 @@ public boolean equals(Object o) {
         return Objects.equals(embeddings, that.embeddings);
     }
 
+    @Override
     public int hashCode() {
         return Objects.hash(embeddings);
     }
+
+    public record Embedding(byte[] values) implements Writeable, ToXContentObject, EmbeddingInt {
+        public static final String EMBEDDING = "embedding";
+
+        public Embedding(StreamInput in) throws IOException {
+            this(in.readByteArray());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeByteArray(values);
+        }
+
+        public static Embedding of(List<Byte> embeddingValuesList) {
+            byte[] embeddingValues = new byte[embeddingValuesList.size()];
+            for (int i = 0; i < embeddingValuesList.size(); i++) {
+                embeddingValues[i] = embeddingValuesList.get(i);
+            }
+            return new Embedding(embeddingValues);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+
+            builder.startArray(EMBEDDING);
+            for (byte value : values) {
+                builder.value(value);
+            }
+            builder.endArray();
+
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+
+        private float[] toFloatArray() {
+            float[] floatArray = new float[values.length];
+            for (int i = 0; i < values.length; i++) {
+                floatArray[i] = ((Byte) values[i]).floatValue();
+            }
+            return floatArray;
+        }
+
+        private double[] toDoubleArray() {
+            double[] doubleArray = new double[values.length];
+            for (int i = 0; i < values.length; i++) {
+                doubleArray[i] = ((Byte) values[i]).floatValue();
+            }
+            return doubleArray;
+        }
+
+        @Override
+        public int getSize() {
+            return values().length;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Embedding embedding = (Embedding) o;
+            return Arrays.equals(values, embedding.values);
+        }
+
+        @Override
+        public int hashCode() {
+            return Arrays.hashCode(values);
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java
index 7652551d9ca1d..152e10e82d5ba 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java
@@ -10,19 +10,24 @@
 package org.elasticsearch.xpack.core.inference.results;
 
 import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 /**
@@ -42,12 +47,12 @@
  *     ]
  * }
  */
-public record TextEmbeddingResults(List<FloatEmbedding> embeddings) implements InferenceServiceResults, TextEmbedding, EmbeddingResults {
+public record TextEmbeddingResults(List<Embedding> embeddings) implements InferenceServiceResults, TextEmbedding {
     public static final String NAME = "text_embedding_service_results";
     public static final String TEXT_EMBEDDING = TaskType.TEXT_EMBEDDING.toString();
 
     public TextEmbeddingResults(StreamInput in) throws IOException {
-        this(in.readCollectionAsList(FloatEmbedding::new));
+        this(in.readCollectionAsList(Embedding::new));
     }
 
     @SuppressWarnings("deprecation")
@@ -55,16 +60,16 @@ public TextEmbeddingResults(StreamInput in) throws IOException {
         this(
             legacyTextEmbeddingResults.embeddings()
                 .stream()
-                .map(embedding -> new FloatEmbedding(embedding.values()))
+                .map(embedding -> new Embedding(embedding.values()))
                 .collect(Collectors.toList())
         );
     }
 
     public static TextEmbeddingResults of(List<? extends InferenceResults> results) {
-        List<FloatEmbedding> embeddings = new ArrayList<>(results.size());
+        List<Embedding> embeddings = new ArrayList<>(results.size());
         for (InferenceResults result : results) {
             if (result instanceof org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults embeddingResult) {
-                embeddings.add(FloatEmbedding.of(embeddingResult));
+                embeddings.add(Embedding.of(embeddingResult));
             } else if (result instanceof org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults errorResult) {
                 if (errorResult.getException() instanceof ElasticsearchStatusException statusException) {
                     throw statusException;
@@ -92,7 +97,7 @@ public int getFirstEmbeddingSize() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startArray(TEXT_EMBEDDING);
-        for (Embedding<?> embedding : embeddings) {
+        for (Embedding embedding : embeddings) {
             embedding.toXContent(builder, params);
         }
         builder.endArray();
@@ -126,7 +131,7 @@ public List<? extends InferenceResults> transformToCoordinationFormat() {
     @SuppressWarnings("deprecation")
     public List<? extends InferenceResults> transformToLegacyFormat() {
         var legacyEmbedding = new LegacyTextEmbeddingResults(
-            embeddings.stream().map(embedding -> new LegacyTextEmbeddingResults.Embedding(embedding.asFloatArray())).toList()
+            embeddings.stream().map(embedding -> new LegacyTextEmbeddingResults.Embedding(embedding.values)).toList()
         );
 
         return List.of(legacyEmbedding);
@@ -140,12 +145,86 @@ public Map<String, Object> asMap() {
     }
 
     @Override
-    public List<FloatEmbedding> embeddings() {
-        return embeddings;
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TextEmbeddingResults that = (TextEmbeddingResults) o;
+        return Objects.equals(embeddings, that.embeddings);
     }
 
     @Override
-    public EmbeddingType embeddingType() {
-        return EmbeddingType.FLOAT;
+    public int hashCode() {
+        return Objects.hash(embeddings);
+    }
+
+    public record Embedding(float[] values) implements Writeable, ToXContentObject, EmbeddingInt {
+        public static final String EMBEDDING = "embedding";
+
+        public Embedding(StreamInput in) throws IOException {
+            this(in.readFloatArray());
+        }
+
+        public static Embedding of(org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults embeddingResult) {
+            float[] embeddingAsArray = embeddingResult.getInferenceAsFloat();
+            return new Embedding(embeddingAsArray);
+        }
+
+        public static Embedding of(List<Float> embeddingValuesList) {
+            float[] embeddingValues = new float[embeddingValuesList.size()];
+            for (int i = 0; i < embeddingValuesList.size(); i++) {
+                embeddingValues[i] = embeddingValuesList.get(i);
+            }
+            return new Embedding(embeddingValues);
+        }
+
+        @Override
+        public int getSize() {
+            return values.length;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeFloatArray(values);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+
+            builder.startArray(EMBEDDING);
+            for (float value : values) {
+                builder.value(value);
+            }
+            builder.endArray();
+
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+
+        private double[] asDoubleArray() {
+            double[] doubles = new double[values.length];
+            for (int i = 0; i < values.length; i++) {
+                doubles[i] = values[i];
+            }
+            return doubles;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Embedding embedding = (Embedding) o;
+            return Arrays.equals(values, embedding.values);
+        }
+
+        @Override
+        public int hashCode() {
+            return Arrays.hashCode(values);
+        }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingUtils.java
index d45a0d9731fc9..4c68d02264457 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingUtils.java
@@ -19,7 +19,7 @@ public class TextEmbeddingUtils {
      * @return the size of the text embedding
      * @throws IllegalStateException if the list of embeddings is empty
      */
-    public static int getFirstEmbeddingSize(List<Embedding<?>> embeddings) throws IllegalStateException {
+    public static int getFirstEmbeddingSize(List<EmbeddingInt> embeddings) throws IllegalStateException {
         if (embeddings.isEmpty()) {
             throw new IllegalStateException("Embeddings list is empty");
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java
index f7fe2d0f6491a..a3fb956c3252d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.xpack.core.ml.inference.results.PyTorchPassThroughResults;
 import org.elasticsearch.xpack.core.ml.inference.results.QuestionAnsweringInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.results.RegressionInferenceResults;
-import org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingByteResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextSimilarityInferenceResults;
@@ -654,9 +653,6 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         );
         namedWriteables.add(new NamedWriteableRegistry.Entry(InferenceResults.class, TextExpansionResults.NAME, TextExpansionResults::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(InferenceResults.class, TextEmbeddingResults.NAME, TextEmbeddingResults::new));
-        namedWriteables.add(
-            new NamedWriteableRegistry.Entry(InferenceResults.class, TextEmbeddingByteResults.NAME, TextEmbeddingByteResults::new)
-        );
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(
                 InferenceResults.class,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingByteResults.java
deleted file mode 100644
index 4871ba208aa6a..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingByteResults.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.ml.inference.results;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Objects;
-
-public class TextEmbeddingByteResults extends NlpInferenceResults {
-
-    public static final String NAME = "text_embedding_byte_result";
-
-    private final String resultsField;
-    private final byte[] inference;
-
-    public TextEmbeddingByteResults(String resultsField, byte[] inference, boolean isTruncated) {
-        super(isTruncated);
-        this.inference = inference;
-        this.resultsField = resultsField;
-    }
-
-    public TextEmbeddingByteResults(StreamInput in) throws IOException {
-        super(in);
-        inference = in.readByteArray();
-        resultsField = in.readString();
-    }
-
-    public String getResultsField() {
-        return resultsField;
-    }
-
-    public byte[] getInference() {
-        return inference;
-    }
-
-    public float[] getInferenceAsFloat() {
-        float[] floatArray = new float[inference.length];
-        for (int i = 0; i < inference.length; i++) {
-            floatArray[i] = inference[i];
-        }
-        return floatArray;
-    }
-
-    @Override
-    void doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(resultsField, inference);
-    }
-
-    @Override
-    public String getWriteableName() {
-        return NAME;
-    }
-
-    @Override
-    void doWriteTo(StreamOutput out) throws IOException {
-        out.writeByteArray(inference);
-        out.writeString(resultsField);
-    }
-
-    @Override
-    void addMapFields(Map<String, Object> map) {
-        map.put(resultsField, inference);
-    }
-
-    @Override
-    public Map<String, Object> asMap(String outputField) {
-        var map = super.asMap(outputField);
-        map.put(outputField, inference);
-        return map;
-    }
-
-    @Override
-    public Object predictedValue() {
-        throw new UnsupportedOperationException("[" + NAME + "] does not support a single predicted value");
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        if (super.equals(o) == false) return false;
-        TextEmbeddingByteResults that = (TextEmbeddingByteResults) o;
-        return Objects.equals(resultsField, that.resultsField) && Arrays.equals(inference, that.inference);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), resultsField, Arrays.hashCode(inference));
-    }
-}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingByteResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingByteResultsTests.java
deleted file mode 100644
index 7ea6e531fcbd2..0000000000000
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingByteResultsTests.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.ml.inference.results;
-
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.ingest.IngestDocument;
-
-import java.util.Map;
-
-import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig.DEFAULT_RESULTS_FIELD;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
-
-public class TextEmbeddingByteResultsTests extends InferenceResultsTestCase<TextEmbeddingByteResults> {
-
-    public static TextEmbeddingByteResults createRandomResults() {
-        int columns = randomIntBetween(1, 10);
-        var arr = new byte[columns];
-        for (int i = 0; i < columns; i++) {
-            arr[i] = randomByte();
-        }
-
-        return new TextEmbeddingByteResults(DEFAULT_RESULTS_FIELD, arr, randomBoolean());
-    }
-
-    @Override
-    protected Writeable.Reader<TextEmbeddingByteResults> instanceReader() {
-        return TextEmbeddingByteResults::new;
-    }
-
-    @Override
-    protected TextEmbeddingByteResults createTestInstance() {
-        return createRandomResults();
-    }
-
-    @Override
-    protected TextEmbeddingByteResults mutateInstance(TextEmbeddingByteResults instance) {
-        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
-    }
-
-    public void testAsMap() {
-        TextEmbeddingByteResults testInstance = createTestInstance();
-        Map<String, Object> asMap = testInstance.asMap();
-        int size = testInstance.isTruncated ? 2 : 1;
-        assertThat(asMap.keySet(), hasSize(size));
-        assertArrayEquals(testInstance.getInference(), (byte[]) asMap.get(DEFAULT_RESULTS_FIELD));
-        if (testInstance.isTruncated) {
-            assertThat(asMap.get("is_truncated"), is(true));
-        }
-    }
-
-    @Override
-    void assertFieldValues(TextEmbeddingByteResults createdInstance, IngestDocument document, String parentField, String resultsField) {
-        assertArrayEquals(document.getFieldValue(parentField + resultsField, byte[].class), createdInstance.getInference());
-    }
-}
diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java
index 9093598a8f6ff..bb18b71eb3fea 100644
--- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java
+++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResults;
 
@@ -138,14 +137,14 @@ public void chunkedInfer(
         }
 
         private TextEmbeddingResults makeResults(List<String> input, int dimensions) {
-            List<FloatEmbedding> embeddings = new ArrayList<>();
+            List<TextEmbeddingResults.Embedding> embeddings = new ArrayList<>();
             for (int i = 0; i < input.size(); i++) {
                 double[] doubleEmbeddings = generateEmbedding(input.get(i), dimensions);
                 List<Float> floatEmbeddings = new ArrayList<>(dimensions);
                 for (int j = 0; j < dimensions; j++) {
                     floatEmbeddings.add((float) doubleEmbeddings[j]);
                 }
-                embeddings.add(FloatEmbedding.of(floatEmbeddings));
+                embeddings.add(TextEmbeddingResults.Embedding.of(floatEmbeddings));
             }
             return new TextEmbeddingResults(embeddings);
         }
diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java
index c39999377ae54..05e85334cff5a 100644
--- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java
+++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults;
-import org.elasticsearch.xpack.core.inference.results.SparseEmbedding;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults;
 import org.elasticsearch.xpack.core.ml.search.WeightedToken;
@@ -128,13 +127,13 @@ public void chunkedInfer(
         }
 
         private SparseEmbeddingResults makeResults(List<String> input) {
-            var embeddings = new ArrayList<SparseEmbedding>();
+            var embeddings = new ArrayList<SparseEmbeddingResults.Embedding>();
             for (int i = 0; i < input.size(); i++) {
-                var tokens = new ArrayList<SparseEmbedding.WeightedToken>();
+                var tokens = new ArrayList<WeightedToken>();
                 for (int j = 0; j < 5; j++) {
-                    tokens.add(new SparseEmbedding.WeightedToken("feature_" + j, generateEmbedding(input.get(i), j)));
+                    tokens.add(new WeightedToken("feature_" + j, generateEmbedding(input.get(i), j)));
                 }
-                embeddings.add(new SparseEmbedding(tokens, false));
+                embeddings.add(new SparseEmbeddingResults.Embedding(tokens, false));
             }
             return new SparseEmbeddingResults(embeddings);
         }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunker.java
index 6866b04f2347e..77d03ac660952 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunker.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunker.java
@@ -7,29 +7,19 @@
 
 package org.elasticsearch.xpack.inference.common;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xpack.core.inference.results.ByteEmbedding;
-import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults;
-import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingByteResults;
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingFloatResults;
-import org.elasticsearch.xpack.core.inference.results.Embedding;
-import org.elasticsearch.xpack.core.inference.results.EmbeddingChunk;
-import org.elasticsearch.xpack.core.inference.results.EmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
-import org.elasticsearch.xpack.core.inference.results.SparseEmbedding;
+import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 /**
@@ -45,8 +35,6 @@
  */
 public class EmbeddingRequestChunker {
 
-    private static final Logger logger = LogManager.getLogger(EmbeddingRequestChunker.class);
-
     public static final int DEFAULT_WORDS_PER_CHUNK = 250;
     public static final int DEFAULT_CHUNK_OVERLAP = 100;
 
@@ -57,12 +45,10 @@ public class EmbeddingRequestChunker {
     private final int chunkOverlap;
 
     private List<List<String>> chunkedInputs;
-    private List<AtomicArray<List<? extends Embedding<?>>>> results;
+    private List<AtomicArray<List<TextEmbeddingResults.Embedding>>> results;
     private AtomicArray<ErrorChunkedInferenceResults> errors;
     private ActionListener<List<ChunkedInferenceServiceResults>> finalListener;
 
-    private AtomicReference<EmbeddingResults.EmbeddingType> firstResultType = new AtomicReference<>();
-
     public EmbeddingRequestChunker(List<String> inputs, int maxNumberOfInputsPerBatch) {
         this.maxNumberOfInputsPerBatch = maxNumberOfInputsPerBatch;
         this.wordsPerChunk = DEFAULT_WORDS_PER_CHUNK;
@@ -174,14 +160,14 @@ private class DebatchingListener implements ActionListener<InferenceServiceResul
 
         @Override
         public void onResponse(InferenceServiceResults inferenceServiceResults) {
-            if (inferenceServiceResults instanceof EmbeddingResults embeddingResults) {
+            if (inferenceServiceResults instanceof TextEmbeddingResults textEmbeddingResults) { // TODO byte embeddings
                 int numRequests = positions.stream().mapToInt(SubBatchPositionsAndCount::embeddingCount).sum();
-                if (numRequests != embeddingResults.embeddings().size()) {
+                if (numRequests != textEmbeddingResults.embeddings().size()) {
                     onFailure(
                         new ElasticsearchStatusException(
                             "Error the number of embedding responses [{}] does not equal the number of " + "requests [{}]",
                             RestStatus.BAD_REQUEST,
-                            embeddingResults.embeddings().size(),
+                            textEmbeddingResults.embeddings().size(),
                             numRequests
                         )
                     );
@@ -191,24 +177,9 @@ public void onResponse(InferenceServiceResults inferenceServiceResults) {
                 int start = 0;
                 for (var pos : positions) {
                     results.get(pos.inputIndex())
-                        .setOnce(pos.chunkIndex(), embeddingResults.embeddings().subList(start, start + pos.embeddingCount()));
+                        .setOnce(pos.chunkIndex(), textEmbeddingResults.embeddings().subList(start, start + pos.embeddingCount()));
                     start += pos.embeddingCount();
                 }
-
-                if (firstResultType.compareAndSet(null, embeddingResults.embeddingType()) == false) {
-                    // firstResultType is set so check the latest matches the first
-                    if (firstResultType.get().equals(embeddingResults.embeddingType()) == false) {
-                        onFailure(
-                            new ElasticsearchStatusException(
-                                "The embedding response types are different. [{}] does not match the first response type [{}]",
-                                RestStatus.BAD_REQUEST,
-                                inferenceServiceResults.getClass().getSimpleName(),
-                                firstResultType.get().matchedClass().getSimpleName()
-                            )
-                        );
-                        return;
-                    }
-                }
             }
 
             if (resultCount.incrementAndGet() == totalNumberOfRequests) {
@@ -234,118 +205,32 @@ private void sendResponse() {
                 if (errors.get(i) != null) {
                     response.add(errors.get(i));
                 } else {
-                    try {
-                        response.add(mergeResults(firstResultType.get(), chunkedInputs.get(i), results.get(i)));
-                    } catch (Exception e) {
-                        response.add(new ErrorChunkedInferenceResults(e));
-                    }
+                    response.add(merge(chunkedInputs.get(i), results.get(i)));
                 }
             }
 
             finalListener.onResponse(response);
         }
 
-        ChunkedInferenceServiceResults mergeResults(
-            EmbeddingResults.EmbeddingType embeddingType,
-            List<String> chunks,
-            AtomicArray<List<? extends Embedding<?>>> embeddings
-        ) {
-            return switch (embeddingType) {
-                case FLOAT -> mergeFloatResults(chunks, embeddings);
-                case BYTE -> mergeByteResults(chunks, embeddings);
-                case SPARSE -> mergeSparseResults(chunks, embeddings);
-            };
-        }
-
-        private ChunkedTextEmbeddingFloatResults mergeFloatResults(
+        private ChunkedTextEmbeddingFloatResults merge(
             List<String> chunks,
-            AtomicArray<List<? extends Embedding<?>>> debatchedResults
+            AtomicArray<List<TextEmbeddingResults.Embedding>> debatchedResults
         ) {
-            var all = new ArrayList<FloatEmbedding>();
+            var all = new ArrayList<TextEmbeddingResults.Embedding>();
             for (int i = 0; i < debatchedResults.length(); i++) {
                 var subBatch = debatchedResults.get(i);
-                for (var result : subBatch) {
-                    if (result instanceof FloatEmbedding fe) {
-                        all.add(fe);
-                    } else {
-                        var message = "Unexpected embedding result type ["
-                            + result.getClass().getSimpleName()
-                            + "], expected a float embedding";
-                        logger.error(message);
-                        throw new IllegalStateException(message);
-                    }
-                }
+                all.addAll(subBatch);
             }
 
             assert chunks.size() == all.size();
 
-            var embeddingChunks = new ArrayList<EmbeddingChunk<FloatEmbedding.FloatArrayWrapper>>();
+            var embeddingChunks = new ArrayList<ChunkedTextEmbeddingFloatResults.EmbeddingChunk>();
             for (int i = 0; i < chunks.size(); i++) {
-                embeddingChunks.add(new EmbeddingChunk<>(chunks.get(i), all.get(i)));
+                embeddingChunks.add(new ChunkedTextEmbeddingFloatResults.EmbeddingChunk(chunks.get(i), all.get(i).values()));
             }
 
             return new ChunkedTextEmbeddingFloatResults(embeddingChunks);
         }
-
-        private ChunkedTextEmbeddingByteResults mergeByteResults(
-            List<String> chunks,
-            AtomicArray<List<? extends Embedding<?>>> debatchedResults
-        ) {
-            var all = new ArrayList<ByteEmbedding>();
-            for (int i = 0; i < debatchedResults.length(); i++) {
-                var subBatch = debatchedResults.get(i);
-                for (var result : subBatch) {
-                    if (result instanceof ByteEmbedding be) {
-                        all.add(be);
-                    } else {
-                        var message = "Unexpected embedding result type ["
-                            + result.getClass().getSimpleName()
-                            + "], expected a byte embedding";
-                        logger.error(message);
-                        throw new IllegalStateException(message);
-                    }
-                }
-            }
-
-            assert chunks.size() == all.size();
-
-            var embeddingChunks = new ArrayList<EmbeddingChunk<ByteEmbedding.ByteArrayWrapper>>();
-            for (int i = 0; i < chunks.size(); i++) {
-                embeddingChunks.add(new EmbeddingChunk<>(chunks.get(i), all.get(i)));
-            }
-
-            return new ChunkedTextEmbeddingByteResults(embeddingChunks, false);
-        }
-
-        private ChunkedSparseEmbeddingResults mergeSparseResults(
-            List<String> chunks,
-            AtomicArray<List<? extends Embedding<?>>> debatchedResults
-        ) {
-            var all = new ArrayList<SparseEmbedding>();
-            for (int i = 0; i < debatchedResults.length(); i++) {
-                var subBatch = debatchedResults.get(i);
-                for (var result : subBatch) {
-                    if (result instanceof SparseEmbedding se) {
-                        all.add(se);
-                    } else {
-                        var message = "Unexpected embedding result type ["
-                            + result.getClass().getSimpleName()
-                            + "], expected a byte embedding";
-                        logger.error(message);
-                        throw new IllegalStateException(message);
-                    }
-                }
-            }
-
-            assert chunks.size() == all.size();
-
-            var embeddingChunks = new ArrayList<EmbeddingChunk<SparseEmbedding.WeightedTokens>>();
-            for (int i = 0; i < chunks.size(); i++) {
-                embeddingChunks.add(new EmbeddingChunk<>(chunks.get(i), all.get(i)));
-            }
-
-            return ChunkedSparseEmbeddingResults.of(embeddingChunks);
-        }
     }
 
     public record BatchRequest(List<SubBatch> subBatches) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java
index 7353f9cf1f94e..f787c6337d646 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java
@@ -18,8 +18,6 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xpack.core.inference.results.ByteEmbedding;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
@@ -190,10 +188,11 @@ private static InferenceServiceResults parseByteEmbeddingsArray(XContentParser p
         return new TextEmbeddingByteResults(embeddingList);
     }
 
-    private static ByteEmbedding parseByteArrayEntry(XContentParser parser) throws IOException {
+    private static TextEmbeddingByteResults.Embedding parseByteArrayEntry(XContentParser parser) throws IOException {
         XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
         List<Byte> embeddingValuesList = XContentParserUtils.parseList(parser, CohereEmbeddingsResponseEntity::parseEmbeddingInt8Entry);
-        return ByteEmbedding.of(embeddingValuesList);
+
+        return TextEmbeddingByteResults.Embedding.of(embeddingValuesList);
     }
 
     private static Byte parseEmbeddingInt8Entry(XContentParser parser) throws IOException {
@@ -217,10 +216,10 @@ private static InferenceServiceResults parseFloatEmbeddingsArray(XContentParser
         return new TextEmbeddingResults(embeddingList);
     }
 
-    private static FloatEmbedding parseFloatArrayEntry(XContentParser parser) throws IOException {
+    private static TextEmbeddingResults.Embedding parseFloatArrayEntry(XContentParser parser) throws IOException {
         XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
-        List<Float> embeddingValues = XContentParserUtils.parseList(parser, CohereEmbeddingsResponseEntity::parseEmbeddingFloatEntry);
-        return FloatEmbedding.of(embeddingValues);
+        List<Float> embeddingValuesList = XContentParserUtils.parseList(parser, CohereEmbeddingsResponseEntity::parseEmbeddingFloatEntry);
+        return TextEmbeddingResults.Embedding.of(embeddingValuesList);
     }
 
     private static Float parseEmbeddingFloatEntry(XContentParser parser) throws IOException {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntity.java
index a18ac3e8e466b..270a981a6998d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntity.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xpack.core.inference.results.SparseEmbedding;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
+import org.elasticsearch.xpack.core.ml.search.WeightedToken;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.request.Request;
 
@@ -65,7 +65,7 @@ public static SparseEmbeddingResults fromResponse(Request request, HttpResult re
             moveToFirstToken(jsonParser);
 
             var truncationResults = request.getTruncationInfo();
-            List<SparseEmbedding> parsedEmbeddings = XContentParserUtils.parseList(
+            List<SparseEmbeddingResults.Embedding> parsedEmbeddings = XContentParserUtils.parseList(
                 jsonParser,
                 (parser, index) -> HuggingFaceElserResponseEntity.parseExpansionResult(truncationResults, parser, index)
             );
@@ -78,24 +78,26 @@ public static SparseEmbeddingResults fromResponse(Request request, HttpResult re
         }
     }
 
-    private static SparseEmbedding parseExpansionResult(boolean[] truncationResults, XContentParser parser, int index) throws IOException {
+    private static SparseEmbeddingResults.Embedding parseExpansionResult(boolean[] truncationResults, XContentParser parser, int index)
+        throws IOException {
         XContentParser.Token token = parser.currentToken();
         XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
 
-        List<SparseEmbedding.WeightedToken> weightedTokens = new ArrayList<>();
+        List<WeightedToken> weightedTokens = new ArrayList<>();
         token = parser.nextToken();
         while (token != null && token != XContentParser.Token.END_OBJECT) {
             XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
             var floatToken = parser.nextToken();
             XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, floatToken, parser);
 
-            weightedTokens.add(new SparseEmbedding.WeightedToken(parser.currentName(), parser.floatValue()));
+            weightedTokens.add(new WeightedToken(parser.currentName(), parser.floatValue()));
+
             token = parser.nextToken();
         }
 
         // prevent an out of bounds if for some reason the truncation list is smaller than the results
         var isTruncated = truncationResults != null && index < truncationResults.length && truncationResults[index];
-        return new SparseEmbedding(weightedTokens, isTruncated);
+        return new SparseEmbeddingResults.Embedding(weightedTokens, isTruncated);
     }
 
     private HuggingFaceElserResponseEntity() {}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java
index 53a2c3e9c7f00..a3e06b3c2075a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.request.Request;
@@ -93,7 +92,7 @@ public static TextEmbeddingResults fromResponse(Request request, HttpResult resp
      * <a href="https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2">sentence-transformers/all-MiniLM-L12-v2</a>
      */
     private static TextEmbeddingResults parseArrayFormat(XContentParser parser) throws IOException {
-        List<FloatEmbedding> embeddingList = XContentParserUtils.parseList(
+        List<TextEmbeddingResults.Embedding> embeddingList = XContentParserUtils.parseList(
             parser,
             HuggingFaceEmbeddingsResponseEntity::parseEmbeddingEntry
         );
@@ -140,7 +139,7 @@ private static TextEmbeddingResults parseArrayFormat(XContentParser parser) thro
     private static TextEmbeddingResults parseObjectFormat(XContentParser parser) throws IOException {
         positionParserAtTokenAfterField(parser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-        List<FloatEmbedding> embeddingList = XContentParserUtils.parseList(
+        List<TextEmbeddingResults.Embedding> embeddingList = XContentParserUtils.parseList(
             parser,
             HuggingFaceEmbeddingsResponseEntity::parseEmbeddingEntry
         );
@@ -148,10 +147,11 @@ private static TextEmbeddingResults parseObjectFormat(XContentParser parser) thr
         return new TextEmbeddingResults(embeddingList);
     }
 
-    private static FloatEmbedding parseEmbeddingEntry(XContentParser parser) throws IOException {
+    private static TextEmbeddingResults.Embedding parseEmbeddingEntry(XContentParser parser) throws IOException {
         XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
+
         List<Float> embeddingValuesList = XContentParserUtils.parseList(parser, HuggingFaceEmbeddingsResponseEntity::parseEmbeddingList);
-        return FloatEmbedding.of(embeddingValuesList);
+        return TextEmbeddingResults.Embedding.of(embeddingValuesList);
     }
 
     private static float parseEmbeddingList(XContentParser parser) throws IOException {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java
index 18d4e78cf0bfe..39b97014c3619 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.request.Request;
@@ -84,7 +83,7 @@ public static TextEmbeddingResults fromResponse(Request request, HttpResult resp
 
             positionParserAtTokenAfterField(jsonParser, "data", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-            List<FloatEmbedding> embeddingList = XContentParserUtils.parseList(
+            List<TextEmbeddingResults.Embedding> embeddingList = XContentParserUtils.parseList(
                 jsonParser,
                 OpenAiEmbeddingsResponseEntity::parseEmbeddingObject
             );
@@ -93,7 +92,7 @@ public static TextEmbeddingResults fromResponse(Request request, HttpResult resp
         }
     }
 
-    private static FloatEmbedding parseEmbeddingObject(XContentParser parser) throws IOException {
+    private static TextEmbeddingResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException {
         XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
 
         positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE);
@@ -102,7 +101,7 @@ private static FloatEmbedding parseEmbeddingObject(XContentParser parser) throws
         // parse and discard the rest of the object
         consumeUntilObjectEnd(parser);
 
-        return FloatEmbedding.of(embeddingValuesList);
+        return TextEmbeddingResults.Embedding.of(embeddingValuesList);
     }
 
     private static float parseEmbeddingList(XContentParser parser) throws IOException {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunkerTests.java
index 2dd9a3fde30d1..164f975cc464f 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunkerTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunkerTests.java
@@ -10,15 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.inference.ChunkedInferenceServiceResults;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.inference.results.ByteEmbedding;
-import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults;
-import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingByteResults;
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingFloatResults;
 import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
-import org.elasticsearch.xpack.core.inference.results.SparseEmbedding;
-import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
-import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 
 import java.util.ArrayList;
@@ -26,7 +19,6 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.startsWith;
@@ -165,7 +157,7 @@ public void testLongInputChunkedOverMultipleBatches() {
         }
     }
 
-    public void testMergingListenerFloatEmbedding() {
+    public void testMergingListener() {
         int batchSize = 5;
         int chunkSize = 20;
         int overlap = 0;
@@ -177,7 +169,7 @@ public void testMergingListenerFloatEmbedding() {
         for (int i = 0; i < numberOfWordsInPassage; i++) {
             passageBuilder.append("passage_input").append(i).append(" "); // chunk on whitespace
         }
-        List<String> inputs = List.of("1st small", passageBuilder.toString());
+        List<String> inputs = List.of("1st small", passageBuilder.toString(), "2nd small", "3rd small");
 
         var finalListener = testListener();
         var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap).batchRequestsWithListeners(finalListener);
@@ -185,22 +177,22 @@ public void testMergingListenerFloatEmbedding() {
 
         // 4 inputs in 2 batches
         {
-            var embeddings = new ArrayList<FloatEmbedding>();
+            var embeddings = new ArrayList<TextEmbeddingResults.Embedding>();
             for (int i = 0; i < batchSize; i++) {
-                embeddings.add(FloatEmbedding.of(List.of(randomFloat())));
+                embeddings.add(new TextEmbeddingResults.Embedding(new float[] { randomFloat() }));
             }
             batches.get(0).listener().onResponse(new TextEmbeddingResults(embeddings));
         }
         {
-            var embeddings = new ArrayList<FloatEmbedding>();
-            for (int i = 0; i < 2; i++) { // 2 requests in the 2nd batch
-                embeddings.add(FloatEmbedding.of(List.of(randomFloat())));
+            var embeddings = new ArrayList<TextEmbeddingResults.Embedding>();
+            for (int i = 0; i < 4; i++) { // 4 requests in the 2nd batch
+                embeddings.add(new TextEmbeddingResults.Embedding(new float[] { randomFloat() }));
             }
             batches.get(1).listener().onResponse(new TextEmbeddingResults(embeddings));
         }
 
         assertNotNull(finalListener.results);
-        assertThat(finalListener.results, hasSize(2));
+        assertThat(finalListener.results, hasSize(4));
         {
             var chunkedResult = finalListener.results.get(0);
             assertThat(chunkedResult, instanceOf(ChunkedTextEmbeddingFloatResults.class));
@@ -221,147 +213,22 @@ public void testMergingListenerFloatEmbedding() {
             assertThat(chunkedFloatResult.chunks().get(4).matchedText(), startsWith(" passage_input80 "));
             assertThat(chunkedFloatResult.chunks().get(5).matchedText(), startsWith(" passage_input100 "));
         }
-    }
-
-    public void testMergingListenerSparseEmbedding() {
-        int batchSize = 5;
-        int chunkSize = 20;
-        int overlap = 0;
-        // passage will be chunked into batchSize + 1 parts
-        // and spread over 2 batch requests
-        int numberOfWordsInPassage = (chunkSize * batchSize) + 5;
-
-        var passageBuilder = new StringBuilder();
-        for (int i = 0; i < numberOfWordsInPassage; i++) {
-            passageBuilder.append("passage_input").append(i).append(" "); // chunk on whitespace
-        }
-        List<String> inputs = List.of("1st small", passageBuilder.toString());
-
-        var finalListener = testListener();
-        var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap).batchRequestsWithListeners(finalListener);
-        assertThat(batches, hasSize(2));
-
-        // 4 inputs in 2 batches
         {
-            var embeddings = new ArrayList<SparseEmbedding>();
-            for (int i = 0; i < batchSize; i++) {
-                embeddings.add(new SparseEmbedding(List.of(new SparseEmbedding.WeightedToken("a", 1.0f)), false));
-            }
-            batches.get(0).listener().onResponse(new SparseEmbeddingResults(embeddings));
-        }
-        {
-            var embeddings = new ArrayList<SparseEmbedding>();
-            for (int i = 0; i < 2; i++) { // 2 requests in the 2nd batch
-                embeddings.add(new SparseEmbedding(List.of(new SparseEmbedding.WeightedToken("b", 1.0f)), false));
-            }
-            batches.get(1).listener().onResponse(new SparseEmbeddingResults(embeddings));
-        }
-
-        assertNotNull(finalListener.results);
-        assertThat(finalListener.results, hasSize(2));
-        {
-            var chunkedResult = finalListener.results.get(0);
-            assertThat(chunkedResult, instanceOf(ChunkedSparseEmbeddingResults.class));
-            var chunkedSparseResult = (ChunkedSparseEmbeddingResults) chunkedResult;
-            assertThat(chunkedSparseResult.getChunkedResults(), hasSize(1));
-            assertEquals("1st small", chunkedSparseResult.getChunkedResults().get(0).matchedText());
-        }
-        {
-            // this is the large input split in multiple chunks
-            var chunkedResult = finalListener.results.get(1);
-            assertThat(chunkedResult, instanceOf(ChunkedSparseEmbeddingResults.class));
-            var chunkedSparseResult = (ChunkedSparseEmbeddingResults) chunkedResult;
-            assertThat(chunkedSparseResult.getChunkedResults(), hasSize(6));
-            assertThat(chunkedSparseResult.getChunkedResults().get(0).matchedText(), startsWith("passage_input0 "));
-            assertThat(chunkedSparseResult.getChunkedResults().get(1).matchedText(), startsWith(" passage_input20 "));
-            assertThat(chunkedSparseResult.getChunkedResults().get(2).matchedText(), startsWith(" passage_input40 "));
-            assertThat(chunkedSparseResult.getChunkedResults().get(3).matchedText(), startsWith(" passage_input60 "));
-            assertThat(chunkedSparseResult.getChunkedResults().get(4).matchedText(), startsWith(" passage_input80 "));
-            assertThat(chunkedSparseResult.getChunkedResults().get(5).matchedText(), startsWith(" passage_input100 "));
-        }
-    }
-
-    public void testMergingListenerByteEmbedding() {
-        int batchSize = 5;
-        int chunkSize = 20;
-        int overlap = 0;
-        // passage will be chunked into batchSize + 1 parts
-        // and spread over 2 batch requests
-        int numberOfWordsInPassage = (chunkSize * batchSize) + 5;
-
-        var passageBuilder = new StringBuilder();
-        for (int i = 0; i < numberOfWordsInPassage; i++) {
-            passageBuilder.append("passage_input").append(i).append(" "); // chunk on whitespace
-        }
-        List<String> inputs = List.of("1st small", passageBuilder.toString());
-
-        var finalListener = testListener();
-        var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap).batchRequestsWithListeners(finalListener);
-        assertThat(batches, hasSize(2));
-
-        // 2 inputs in 2 batches
-        {
-            var embeddings = new ArrayList<ByteEmbedding>();
-            for (int i = 0; i < batchSize; i++) {
-                embeddings.add(ByteEmbedding.of(List.of(randomByte())));
-            }
-            batches.get(0).listener().onResponse(new TextEmbeddingByteResults(embeddings));
-        }
-        {
-            var embeddings = new ArrayList<ByteEmbedding>();
-            for (int i = 0; i < 2; i++) { // 2 requests in the 2nd batch
-                embeddings.add(ByteEmbedding.of(List.of(randomByte())));
-            }
-            batches.get(1).listener().onResponse(new TextEmbeddingByteResults(embeddings));
-        }
-
-        assertNotNull(finalListener.results);
-        assertThat(finalListener.results, hasSize(2));
-        {
-            var chunkedResult = finalListener.results.get(0);
-            assertThat(chunkedResult, instanceOf(ChunkedTextEmbeddingByteResults.class));
-            var chunkedByteResult = (ChunkedTextEmbeddingByteResults) chunkedResult;
-            assertThat(chunkedByteResult.chunks(), hasSize(1));
-            assertEquals("1st small", chunkedByteResult.chunks().get(0).matchedText());
+            var chunkedResult = finalListener.results.get(2);
+            assertThat(chunkedResult, instanceOf(ChunkedTextEmbeddingFloatResults.class));
+            var chunkedFloatResult = (ChunkedTextEmbeddingFloatResults) chunkedResult;
+            assertThat(chunkedFloatResult.chunks(), hasSize(1));
+            assertEquals("2nd small", chunkedFloatResult.chunks().get(0).matchedText());
         }
         {
-            // this is the large input split in multiple chunks
-            var chunkedResult = finalListener.results.get(1);
-            assertThat(chunkedResult, instanceOf(ChunkedTextEmbeddingByteResults.class));
-            var chunkedByteResult = (ChunkedTextEmbeddingByteResults) chunkedResult;
-            assertThat(chunkedByteResult.chunks(), hasSize(6));
-            assertThat(chunkedByteResult.chunks().get(0).matchedText(), startsWith("passage_input0 "));
-            assertThat(chunkedByteResult.chunks().get(1).matchedText(), startsWith(" passage_input20 "));
-            assertThat(chunkedByteResult.chunks().get(2).matchedText(), startsWith(" passage_input40 "));
-            assertThat(chunkedByteResult.chunks().get(3).matchedText(), startsWith(" passage_input60 "));
-            assertThat(chunkedByteResult.chunks().get(4).matchedText(), startsWith(" passage_input80 "));
-            assertThat(chunkedByteResult.chunks().get(5).matchedText(), startsWith(" passage_input100 "));
+            var chunkedResult = finalListener.results.get(3);
+            assertThat(chunkedResult, instanceOf(ChunkedTextEmbeddingFloatResults.class));
+            var chunkedFloatResult = (ChunkedTextEmbeddingFloatResults) chunkedResult;
+            assertThat(chunkedFloatResult.chunks(), hasSize(1));
+            assertEquals("3rd small", chunkedFloatResult.chunks().get(0).matchedText());
         }
     }
 
-    public void testDifferentResponseTypes() {
-        List<String> inputs = List.of("one", "two");
-        var result = new AtomicReference<List<ChunkedInferenceServiceResults>>();
-
-        var listener = ActionListener.<List<ChunkedInferenceServiceResults>>wrap(result::set, e -> fail(e.getMessage()));
-
-        var batches = new EmbeddingRequestChunker(inputs, 1, 100, 0).batchRequestsWithListeners(listener);
-        assertThat(batches, hasSize(2));
-
-        batches.get(0).listener().onResponse(new TextEmbeddingByteResults(List.of(ByteEmbedding.of(List.of(randomByte())))));
-        batches.get(1).listener().onResponse(new TextEmbeddingResults(List.of(FloatEmbedding.of(List.of(randomFloat())))));
-
-        assertThat(result.get().get(0), instanceOf(ChunkedTextEmbeddingByteResults.class));
-        assertThat(result.get().get(1), instanceOf(ErrorChunkedInferenceResults.class));
-        assertThat(
-            ((ErrorChunkedInferenceResults) result.get().get(1)).getException().getMessage(),
-            containsString(
-                "The embedding response types are different. [TextEmbeddingResults] does not match the first response type "
-                    + "[TextEmbeddingByteResults]"
-            )
-        );
-    }
-
     public void testListenerErrorsWithWrongNumberOfResponses() {
         List<String> inputs = List.of("1st small", "2nd small", "3rd small");
 
@@ -384,9 +251,9 @@ public void onFailure(Exception e) {
         var batches = new EmbeddingRequestChunker(inputs, 10, 100, 0).batchRequestsWithListeners(listener);
         assertThat(batches, hasSize(1));
 
-        var embeddings = new ArrayList<FloatEmbedding>();
-        embeddings.add(FloatEmbedding.of(List.of(randomFloat())));
-        embeddings.add(FloatEmbedding.of(List.of(randomFloat())));
+        var embeddings = new ArrayList<TextEmbeddingResults.Embedding>();
+        embeddings.add(new TextEmbeddingResults.Embedding(new float[] { randomFloat() }));
+        embeddings.add(new TextEmbeddingResults.Embedding(new float[] { randomFloat() }));
         batches.get(0).listener().onResponse(new TextEmbeddingResults(embeddings));
         assertEquals("Error the number of embedding responses [2] does not equal the number of requests [3]", failureMessage.get());
     }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioEmbeddingsResponseEntityTests.java
index e744b579924c8..41768a6814f36 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioEmbeddingsResponseEntityTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioEmbeddingsResponseEntityTests.java
@@ -9,7 +9,6 @@
 
 import org.apache.http.HttpResponse;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.request.Request;
@@ -56,6 +55,6 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException {
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(0.014539449F, -0.015288644F)))));
+        assertThat(parsedResults.embeddings(), is(List.of(TextEmbeddingResults.Embedding.of(List.of(0.014539449F, -0.015288644F)))));
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntityTests.java
index d0f6f45974d1e..d809635aa4f38 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntityTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntityTests.java
@@ -10,8 +10,6 @@
 import org.apache.http.HttpResponse;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.inference.results.ByteEmbedding;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
@@ -60,7 +58,7 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException {
         MatcherAssert.assertThat(parsedResults, instanceOf(TextEmbeddingResults.class));
         MatcherAssert.assertThat(
             ((TextEmbeddingResults) parsedResults).embeddings(),
-            is(List.of(FloatEmbedding.of(List.of(-0.0018434525F, 0.01777649F))))
+            is(List.of(new TextEmbeddingResults.Embedding(new float[] { -0.0018434525F, 0.01777649F })))
         );
     }
 
@@ -96,7 +94,10 @@ public void testFromResponse_CreatesResultsForASingleItem_ObjectFormat() throws
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        MatcherAssert.assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(-0.0018434525F, 0.01777649F)))));
+        MatcherAssert.assertThat(
+            parsedResults.embeddings(),
+            is(List.of(new TextEmbeddingResults.Embedding(new float[] { -0.0018434525F, 0.01777649F })))
+        );
     }
 
     public void testFromResponse_UsesTheFirstValidEmbeddingsEntry() throws IOException {
@@ -137,7 +138,10 @@ public void testFromResponse_UsesTheFirstValidEmbeddingsEntry() throws IOExcepti
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        MatcherAssert.assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(-0.0018434525F, 0.01777649F)))));
+        MatcherAssert.assertThat(
+            parsedResults.embeddings(),
+            is(List.of(new TextEmbeddingResults.Embedding(new float[] { -0.0018434525F, 0.01777649F })))
+        );
     }
 
     public void testFromResponse_UsesTheFirstValidEmbeddingsEntryInt8_WithInvalidFirst() throws IOException {
@@ -178,7 +182,10 @@ public void testFromResponse_UsesTheFirstValidEmbeddingsEntryInt8_WithInvalidFir
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        MatcherAssert.assertThat(parsedResults.embeddings(), is(List.of(ByteEmbedding.of(List.of((byte) -1, (byte) 0)))));
+        MatcherAssert.assertThat(
+            parsedResults.embeddings(),
+            is(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) -1, (byte) 0 })))
+        );
     }
 
     public void testFromResponse_ParsesBytes() throws IOException {
@@ -213,7 +220,10 @@ public void testFromResponse_ParsesBytes() throws IOException {
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        MatcherAssert.assertThat(parsedResults.embeddings(), is(List.of(ByteEmbedding.of(List.of((byte) -1, (byte) 0)))));
+        MatcherAssert.assertThat(
+            parsedResults.embeddings(),
+            is(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) -1, (byte) 0 })))
+        );
     }
 
     public void testFromResponse_CreatesResultsForMultipleItems() throws IOException {
@@ -252,7 +262,12 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException
 
         MatcherAssert.assertThat(
             parsedResults.embeddings(),
-            is(List.of(FloatEmbedding.of(List.of(-0.0018434525F, 0.01777649F)), FloatEmbedding.of(List.of(-0.123F, 0.123F))))
+            is(
+                List.of(
+                    new TextEmbeddingResults.Embedding(new float[] { -0.0018434525F, 0.01777649F }),
+                    new TextEmbeddingResults.Embedding(new float[] { -0.123F, 0.123F })
+                )
+            )
         );
     }
 
@@ -294,7 +309,12 @@ public void testFromResponse_CreatesResultsForMultipleItems_ObjectFormat() throw
 
         MatcherAssert.assertThat(
             parsedResults.embeddings(),
-            is(List.of(FloatEmbedding.of(List.of(-0.0018434525F, 0.01777649F)), FloatEmbedding.of(List.of(-0.123F, 0.123F))))
+            is(
+                List.of(
+                    new TextEmbeddingResults.Embedding(new float[] { -0.0018434525F, 0.01777649F }),
+                    new TextEmbeddingResults.Embedding(new float[] { -0.123F, 0.123F })
+                )
+            )
         );
     }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntityTests.java
index b69bb37d35c53..238dab5929139 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntityTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntityTests.java
@@ -10,7 +10,6 @@
 import org.apache.http.HttpResponse;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.request.Request;
@@ -38,7 +37,10 @@ public void testFromResponse_CreatesResultsForASingleItem_ArrayFormat() throws I
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(0.014539449F, -0.015288644F)))));
+        assertThat(
+            parsedResults.embeddings(),
+            is(List.of(new TextEmbeddingResults.Embedding(new float[] { 0.014539449F, -0.015288644F })))
+        );
     }
 
     public void testFromResponse_CreatesResultsForASingleItem_ObjectFormat() throws IOException {
@@ -58,7 +60,10 @@ public void testFromResponse_CreatesResultsForASingleItem_ObjectFormat() throws
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(0.014539449F, -0.015288644F)))));
+        assertThat(
+            parsedResults.embeddings(),
+            is(List.of(new TextEmbeddingResults.Embedding(new float[] { 0.014539449F, -0.015288644F })))
+        );
     }
 
     public void testFromResponse_CreatesResultsForMultipleItems_ArrayFormat() throws IOException {
@@ -82,7 +87,12 @@ public void testFromResponse_CreatesResultsForMultipleItems_ArrayFormat() throws
 
         assertThat(
             parsedResults.embeddings(),
-            is(List.of(FloatEmbedding.of(List.of(0.014539449F, -0.015288644F)), FloatEmbedding.of(List.of(0.0123F, -0.0123F))))
+            is(
+                List.of(
+                    new TextEmbeddingResults.Embedding(new float[] { 0.014539449F, -0.015288644F }),
+                    new TextEmbeddingResults.Embedding(new float[] { 0.0123F, -0.0123F })
+                )
+            )
         );
     }
 
@@ -109,7 +119,12 @@ public void testFromResponse_CreatesResultsForMultipleItems_ObjectFormat() throw
 
         assertThat(
             parsedResults.embeddings(),
-            is(List.of(FloatEmbedding.of(List.of(0.014539449F, -0.015288644F)), FloatEmbedding.of(List.of(0.0123F, -0.0123F))))
+            is(
+                List.of(
+                    new TextEmbeddingResults.Embedding(new float[] { 0.014539449F, -0.015288644F }),
+                    new TextEmbeddingResults.Embedding(new float[] { 0.0123F, -0.0123F })
+                )
+            )
         );
     }
 
@@ -245,7 +260,7 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsInt_ArrayFormat() throw
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(1.0F)))));
+        assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingResults.Embedding(new float[] { 1.0F }))));
     }
 
     public void testFromResponse_SucceedsWhenEmbeddingValueIsInt_ObjectFormat() throws IOException {
@@ -264,7 +279,7 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsInt_ObjectFormat() thro
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(1.0F)))));
+        assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingResults.Embedding(new float[] { 1.0F }))));
     }
 
     public void testFromResponse_SucceedsWhenEmbeddingValueIsLong_ArrayFormat() throws IOException {
@@ -281,7 +296,7 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsLong_ArrayFormat() thro
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(4.0294965E10F)))));
+        assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingResults.Embedding(new float[] { 4.0294965E10F }))));
     }
 
     public void testFromResponse_SucceedsWhenEmbeddingValueIsLong_ObjectFormat() throws IOException {
@@ -300,7 +315,7 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsLong_ObjectFormat() thr
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(4.0294965E10F)))));
+        assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingResults.Embedding(new float[] { 4.0294965E10F }))));
     }
 
     public void testFromResponse_FailsWhenEmbeddingValueIsAnObject_ObjectFormat() {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntityTests.java
index 7299f7ab6ef7b..6c38092f509a7 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntityTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntityTests.java
@@ -10,7 +10,6 @@
 import org.apache.http.HttpResponse;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.request.Request;
@@ -50,7 +49,10 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException {
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(0.014539449F, -0.015288644F)))));
+        assertThat(
+            parsedResults.embeddings(),
+            is(List.of(new TextEmbeddingResults.Embedding(new float[] { 0.014539449F, -0.015288644F })))
+        );
     }
 
     public void testFromResponse_CreatesResultsForMultipleItems() throws IOException {
@@ -90,7 +92,12 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException
 
         assertThat(
             parsedResults.embeddings(),
-            is(List.of(FloatEmbedding.of(List.of(0.014539449F, -0.015288644F)), FloatEmbedding.of(List.of(0.0123F, -0.0123F))))
+            is(
+                List.of(
+                    new TextEmbeddingResults.Embedding(new float[] { 0.014539449F, -0.015288644F }),
+                    new TextEmbeddingResults.Embedding(new float[] { 0.0123F, -0.0123F })
+                )
+            )
         );
     }
 
@@ -257,7 +264,7 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsInt() throws IOExceptio
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(1.0F)))));
+        assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingResults.Embedding(new float[] { 1.0F }))));
     }
 
     public void testFromResponse_SucceedsWhenEmbeddingValueIsLong() throws IOException {
@@ -286,7 +293,7 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsLong() throws IOExcepti
             new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
         );
 
-        assertThat(parsedResults.embeddings(), is(List.of(FloatEmbedding.of(List.of(4.0294965E10F)))));
+        assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingResults.Embedding(new float[] { 4.0294965E10F }))));
     }
 
     public void testFromResponse_FailsWhenEmbeddingValueIsAnObject() {
@@ -375,9 +382,9 @@ public void testFieldsInDifferentOrderServer() throws IOException {
             parsedResults.embeddings(),
             is(
                 List.of(
-                    FloatEmbedding.of(List.of(-0.9F, 0.5F, 0.3F)),
-                    FloatEmbedding.of(List.of(0.1F, 0.5F)),
-                    FloatEmbedding.of(List.of(0.5F, 0.5F))
+                    new TextEmbeddingResults.Embedding(new float[] { -0.9F, 0.5F, 0.3F }),
+                    new TextEmbeddingResults.Embedding(new float[] { 0.1F, 0.5F }),
+                    new TextEmbeddingResults.Embedding(new float[] { 0.5F, 0.5F })
                 )
             )
         );
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestInferenceActionTests.java
index eb9cf8324683f..8365ebdfad786 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestInferenceActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/RestInferenceActionTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.test.rest.RestActionTestCase;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
-import org.elasticsearch.xpack.core.inference.results.ByteEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults;
 import org.junit.Before;
 
@@ -76,6 +75,8 @@ public void testUses3SecondTimeoutFromParams() {
     }
 
     private static InferenceAction.Response createResponse() {
-        return new InferenceAction.Response(new TextEmbeddingByteResults(List.of(ByteEmbedding.of(List.of((byte) -1)))));
+        return new InferenceAction.Response(
+            new TextEmbeddingByteResults(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) -1 })))
+        );
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedSparseEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedSparseEmbeddingResultsTests.java
index 392a49be7cd42..073a662c1e8f2 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedSparseEmbeddingResultsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedSparseEmbeddingResultsTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults;
-import org.elasticsearch.xpack.core.inference.results.SparseEmbedding;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResults;
@@ -74,7 +73,7 @@ public void testToXContent_CreatesTheRightJsonForASingleChunk() {
     public void testToXContent_CreatesTheRightJsonForASingleChunk_FromSparseEmbeddingResults() {
         var entity = ChunkedSparseEmbeddingResults.of(
             List.of("text"),
-            new SparseEmbeddingResults(List.of(new SparseEmbedding(List.of(new SparseEmbedding.WeightedToken("token", 0.1f)), false)))
+            new SparseEmbeddingResults(List.of(new SparseEmbeddingResults.Embedding(List.of(new WeightedToken("token", 0.1f)), false)))
         );
 
         assertThat(entity.size(), is(1));
@@ -110,7 +109,7 @@ public void testToXContent_ThrowsWhenInputSizeIsDifferentThanEmbeddings() {
             IllegalArgumentException.class,
             () -> ChunkedSparseEmbeddingResults.of(
                 List.of("text", "text2"),
-                new SparseEmbeddingResults(List.of(new SparseEmbedding(List.of(new SparseEmbedding.WeightedToken("token", 0.1f)), false)))
+                new SparseEmbeddingResults(List.of(new SparseEmbeddingResults.Embedding(List.of(new WeightedToken("token", 0.1f)), false)))
             )
         );
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingByteResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingByteResultsTests.java
index cc1b000644dad..6d6fbe956280a 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingByteResultsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingByteResultsTests.java
@@ -10,9 +10,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.core.inference.results.ByteEmbedding;
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingByteResults;
-import org.elasticsearch.xpack.core.inference.results.EmbeddingChunk;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults;
 
 import java.io.IOException;
@@ -26,7 +24,7 @@ public class ChunkedTextEmbeddingByteResultsTests extends AbstractWireSerializin
 
     public static ChunkedTextEmbeddingByteResults createRandomResults() {
         int numChunks = randomIntBetween(1, 5);
-        var chunks = new ArrayList<EmbeddingChunk<ByteEmbedding.ByteArrayWrapper>>(numChunks);
+        var chunks = new ArrayList<ChunkedTextEmbeddingByteResults.EmbeddingChunk>(numChunks);
 
         for (int i = 0; i < numChunks; i++) {
             chunks.add(createRandomChunk());
@@ -35,25 +33,28 @@ public static ChunkedTextEmbeddingByteResults createRandomResults() {
         return new ChunkedTextEmbeddingByteResults(chunks, randomBoolean());
     }
 
-    private static EmbeddingChunk<ByteEmbedding.ByteArrayWrapper> createRandomChunk() {
+    private static ChunkedTextEmbeddingByteResults.EmbeddingChunk createRandomChunk() {
         int columns = randomIntBetween(1, 10);
         byte[] bytes = new byte[columns];
         for (int i = 0; i < columns; i++) {
             bytes[i] = randomByte();
         }
 
-        return new EmbeddingChunk<>(randomAlphaOfLength(6), new ByteEmbedding(bytes));
+        return new ChunkedTextEmbeddingByteResults.EmbeddingChunk(randomAlphaOfLength(6), bytes);
     }
 
     public void testToXContent_CreatesTheRightJsonForASingleChunk() {
-        var entity = new ChunkedTextEmbeddingByteResults(List.of(new EmbeddingChunk<>("text", ByteEmbedding.of(List.of((byte) 1)))), false);
+        var entity = new ChunkedTextEmbeddingByteResults(
+            List.of(new ChunkedTextEmbeddingByteResults.EmbeddingChunk("text", new byte[] { (byte) 1 })),
+            false
+        );
 
         assertThat(
             entity.asMap(),
             is(
                 Map.of(
                     ChunkedTextEmbeddingByteResults.FIELD_NAME,
-                    List.of(new EmbeddingChunk<>("text", new ByteEmbedding(new byte[] { (byte) 1 })))
+                    List.of(new ChunkedTextEmbeddingByteResults.EmbeddingChunk("text", new byte[] { (byte) 1 }))
                 )
             )
         );
@@ -74,7 +75,7 @@ public void testToXContent_CreatesTheRightJsonForASingleChunk() {
     public void testToXContent_CreatesTheRightJsonForASingleChunk_ForTextEmbeddingByteResults() {
         var entity = ChunkedTextEmbeddingByteResults.of(
             List.of("text"),
-            new TextEmbeddingByteResults(List.of(ByteEmbedding.of(List.of((byte) 1))))
+            new TextEmbeddingByteResults(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 1 })))
         );
 
         assertThat(entity.size(), is(1));
@@ -86,7 +87,7 @@ public void testToXContent_CreatesTheRightJsonForASingleChunk_ForTextEmbeddingBy
             is(
                 Map.of(
                     ChunkedTextEmbeddingByteResults.FIELD_NAME,
-                    List.of(new EmbeddingChunk<>("text", new ByteEmbedding(new byte[] { (byte) 1 })))
+                    List.of(new ChunkedTextEmbeddingByteResults.EmbeddingChunk("text", new byte[] { (byte) 1 }))
                 )
             )
         );
@@ -109,7 +110,7 @@ public void testToXContent_ThrowsWhenInputSizeIsDifferentThanEmbeddings() {
             IllegalArgumentException.class,
             () -> ChunkedTextEmbeddingByteResults.of(
                 List.of("text", "text2"),
-                new TextEmbeddingByteResults(List.of(ByteEmbedding.of(List.of((byte) 1))))
+                new TextEmbeddingByteResults(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 1 })))
             )
         );
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingFloatResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingFloatResultsTests.java
index 461bf793996af..beb75fbfa36a6 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingFloatResultsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingFloatResultsTests.java
@@ -10,8 +10,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingFloatResults;
-import org.elasticsearch.xpack.core.inference.results.EmbeddingChunk;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -20,7 +18,7 @@ public class ChunkedTextEmbeddingFloatResultsTests extends AbstractWireSerializi
 
     public static ChunkedTextEmbeddingFloatResults createRandomResults() {
         int numChunks = randomIntBetween(1, 5);
-        var chunks = new ArrayList<EmbeddingChunk<FloatEmbedding.FloatArrayWrapper>>(numChunks);
+        var chunks = new ArrayList<ChunkedTextEmbeddingFloatResults.EmbeddingChunk>(numChunks);
 
         for (int i = 0; i < numChunks; i++) {
             chunks.add(createRandomChunk());
@@ -29,14 +27,14 @@ public static ChunkedTextEmbeddingFloatResults createRandomResults() {
         return new ChunkedTextEmbeddingFloatResults(chunks);
     }
 
-    private static EmbeddingChunk<FloatEmbedding.FloatArrayWrapper> createRandomChunk() {
+    private static ChunkedTextEmbeddingFloatResults.EmbeddingChunk createRandomChunk() {
         int columns = randomIntBetween(1, 10);
         float[] floats = new float[columns];
         for (int i = 0; i < columns; i++) {
             floats[i] = randomFloat();
         }
 
-        return new EmbeddingChunk<>(randomAlphaOfLength(6), new FloatEmbedding(floats));
+        return new ChunkedTextEmbeddingFloatResults.EmbeddingChunk(randomAlphaOfLength(6), floats);
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingResultsTests.java
index 935180cce5d95..1fc0282b5d96d 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingResultsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/ChunkedTextEmbeddingResultsTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingResults;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults;
 
@@ -98,7 +97,7 @@ public void testToXContent_CreatesTheRightJsonForASingleChunk() {
     public void testToXContent_CreatesTheRightJsonForASingleChunk_FromTextEmbeddingResults() {
         var entity = ChunkedTextEmbeddingResults.of(
             List.of("text"),
-            new TextEmbeddingResults(List.of(FloatEmbedding.of(List.of(0.1f, 0.2f))))
+            new TextEmbeddingResults(List.of(new TextEmbeddingResults.Embedding(new float[] { 0.1f, 0.2f })))
         );
 
         assertThat(entity.size(), is(1));
@@ -141,7 +140,7 @@ public void testToXContent_ThrowsWhenInputSizeIsDifferentThanEmbeddings() {
             IllegalArgumentException.class,
             () -> ChunkedTextEmbeddingResults.of(
                 List.of("text", "text2"),
-                new TextEmbeddingResults(List.of(FloatEmbedding.of(List.of(0.1f, 0.2f))))
+                new TextEmbeddingResults(List.of(new TextEmbeddingResults.Embedding(new float[] { 0.1f, 0.2f })))
             )
         );
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/SparseEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/SparseEmbeddingResultsTests.java
index acc0ef6eed269..dcdbc13f097b6 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/SparseEmbeddingResultsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/SparseEmbeddingResultsTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.core.inference.results.SparseEmbedding;
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults;
 import org.elasticsearch.xpack.core.ml.search.WeightedToken;
@@ -32,7 +31,7 @@ public static SparseEmbeddingResults createRandomResults() {
     }
 
     public static SparseEmbeddingResults createRandomResults(int numEmbeddings, int numTokens) {
-        List<SparseEmbedding> embeddings = new ArrayList<>(numEmbeddings);
+        List<SparseEmbeddingResults.Embedding> embeddings = new ArrayList<>(numEmbeddings);
 
         for (int i = 0; i < numEmbeddings; i++) {
             embeddings.add(createRandomEmbedding(numTokens));
@@ -42,7 +41,7 @@ public static SparseEmbeddingResults createRandomResults(int numEmbeddings, int
     }
 
     public static SparseEmbeddingResults createRandomResults(List<String> input) {
-        List<SparseEmbedding> embeddings = new ArrayList<>(input.size());
+        List<SparseEmbeddingResults.Embedding> embeddings = new ArrayList<>(input.size());
 
         for (String s : input) {
             int numTokens = Strings.tokenizeToStringArray(s, " ").length;
@@ -52,13 +51,13 @@ public static SparseEmbeddingResults createRandomResults(List<String> input) {
         return new SparseEmbeddingResults(embeddings);
     }
 
-    private static SparseEmbedding createRandomEmbedding(int numTokens) {
-        List<SparseEmbedding.WeightedToken> tokenList = new ArrayList<>(numTokens);
+    private static SparseEmbeddingResults.Embedding createRandomEmbedding(int numTokens) {
+        List<WeightedToken> tokenList = new ArrayList<>(numTokens);
         for (int i = 0; i < numTokens; i++) {
-            tokenList.add(new SparseEmbedding.WeightedToken(Integer.toString(i), (float) randomDoubleBetween(0.0, 5.0, false)));
+            tokenList.add(new WeightedToken(Integer.toString(i), (float) randomDoubleBetween(0.0, 5.0, false)));
         }
 
-        return new SparseEmbedding(tokenList, randomBoolean());
+        return new SparseEmbeddingResults.Embedding(tokenList, randomBoolean());
     }
 
     @Override
@@ -79,14 +78,14 @@ protected SparseEmbeddingResults mutateInstance(SparseEmbeddingResults instance)
             int end = randomInt(instance.embeddings().size() - 1);
             return new SparseEmbeddingResults(instance.embeddings().subList(0, end));
         } else {
-            List<SparseEmbedding> embeddings = new ArrayList<>(instance.embeddings());
+            List<SparseEmbeddingResults.Embedding> embeddings = new ArrayList<>(instance.embeddings());
             embeddings.add(createRandomEmbedding(randomIntBetween(0, 20)));
             return new SparseEmbeddingResults(embeddings);
         }
     }
 
     public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOException {
-        var entity = createSparseResult(List.of(createEmbedding(List.of(new SparseEmbedding.WeightedToken("token", 0.1F)), false)));
+        var entity = createSparseResult(List.of(createEmbedding(List.of(new WeightedToken("token", 0.1F)), false)));
         assertThat(entity.asMap(), is(buildExpectationSparseEmbeddings(List.of(new EmbeddingExpectation(Map.of("token", 0.1F), false)))));
         String xContentResult = Strings.toString(entity, true, true);
         assertThat(xContentResult, is("""
@@ -105,14 +104,8 @@ public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOE
     public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws IOException {
         var entity = createSparseResult(
             List.of(
-                new SparseEmbedding(
-                    List.of(new SparseEmbedding.WeightedToken("token", 0.1F), new SparseEmbedding.WeightedToken("token2", 0.2F)),
-                    false
-                ),
-                new SparseEmbedding(
-                    List.of(new SparseEmbedding.WeightedToken("token3", 0.3F), new SparseEmbedding.WeightedToken("token4", 0.4F)),
-                    false
-                )
+                new SparseEmbeddingResults.Embedding(List.of(new WeightedToken("token", 0.1F), new WeightedToken("token2", 0.2F)), false),
+                new SparseEmbeddingResults.Embedding(List.of(new WeightedToken("token3", 0.3F), new WeightedToken("token4", 0.4F)), false)
             )
         );
         assertThat(
@@ -152,8 +145,8 @@ public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws I
     public void testTransformToCoordinationFormat() {
         var results = createSparseResult(
             List.of(
-                createEmbedding(List.of(new SparseEmbedding.WeightedToken("token", 0.1F)), false),
-                createEmbedding(List.of(new SparseEmbedding.WeightedToken("token2", 0.2F)), true)
+                createEmbedding(List.of(new WeightedToken("token", 0.1F)), false),
+                createEmbedding(List.of(new WeightedToken("token2", 0.2F)), true)
             )
         ).transformToCoordinationFormat();
 
@@ -174,16 +167,23 @@ public static Map<String, Object> buildExpectationSparseEmbeddings(List<Embeddin
         return Map.of(
             SparseEmbeddingResults.SPARSE_EMBEDDING,
             embeddings.stream()
-                .map(embedding -> Map.of(SparseEmbedding.EMBEDDING, embedding.tokens, SparseEmbedding.IS_TRUNCATED, embedding.isTruncated))
+                .map(
+                    embedding -> Map.of(
+                        SparseEmbeddingResults.Embedding.EMBEDDING,
+                        embedding.tokens,
+                        SparseEmbeddingResults.Embedding.IS_TRUNCATED,
+                        embedding.isTruncated
+                    )
+                )
                 .toList()
         );
     }
 
-    public static SparseEmbeddingResults createSparseResult(List<SparseEmbedding> embeddings) {
+    public static SparseEmbeddingResults createSparseResult(List<SparseEmbeddingResults.Embedding> embeddings) {
         return new SparseEmbeddingResults(embeddings);
     }
 
-    public static SparseEmbedding createEmbedding(List<SparseEmbedding.WeightedToken> tokensList, boolean isTruncated) {
-        return new SparseEmbedding(tokensList, isTruncated);
+    public static SparseEmbeddingResults.Embedding createEmbedding(List<WeightedToken> tokensList, boolean isTruncated) {
+        return new SparseEmbeddingResults.Embedding(tokensList, isTruncated);
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java
index a07f75ec2c536..a15d6323d1315 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.core.inference.results.ByteEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults;
 
 import java.io.IOException;
@@ -23,7 +22,7 @@
 public class TextEmbeddingByteResultsTests extends AbstractWireSerializingTestCase<TextEmbeddingByteResults> {
     public static TextEmbeddingByteResults createRandomResults() {
         int embeddings = randomIntBetween(1, 10);
-        List<ByteEmbedding> embeddingResults = new ArrayList<>(embeddings);
+        List<TextEmbeddingByteResults.Embedding> embeddingResults = new ArrayList<>(embeddings);
 
         for (int i = 0; i < embeddings; i++) {
             embeddingResults.add(createRandomEmbedding());
@@ -32,7 +31,7 @@ public static TextEmbeddingByteResults createRandomResults() {
         return new TextEmbeddingByteResults(embeddingResults);
     }
 
-    private static ByteEmbedding createRandomEmbedding() {
+    private static TextEmbeddingByteResults.Embedding createRandomEmbedding() {
         int columns = randomIntBetween(1, 10);
         byte[] bytes = new byte[columns];
 
@@ -40,11 +39,11 @@ private static ByteEmbedding createRandomEmbedding() {
             bytes[i] = randomByte();
         }
 
-        return new ByteEmbedding(bytes);
+        return new TextEmbeddingByteResults.Embedding(bytes);
     }
 
     public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOException {
-        var entity = new TextEmbeddingByteResults(List.of(ByteEmbedding.of(List.of((byte) 23))));
+        var entity = new TextEmbeddingByteResults(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23 })));
 
         String xContentResult = Strings.toString(entity, true, true);
         assertThat(xContentResult, is("""
@@ -61,8 +60,10 @@ public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOE
 
     public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws IOException {
         var entity = new TextEmbeddingByteResults(
-            List.of(ByteEmbedding.of(List.of((byte) 23)), ByteEmbedding.of(List.of((byte) 24)))
-
+            List.of(
+                new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23 }),
+                new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 24 })
+            )
         );
 
         String xContentResult = Strings.toString(entity, true, true);
@@ -85,21 +86,24 @@ public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws I
 
     public void testTransformToCoordinationFormat() {
         var results = new TextEmbeddingByteResults(
-            List.of(ByteEmbedding.of(List.of((byte) 23, (byte) 24)), ByteEmbedding.of(List.of((byte) 25, (byte) 26)))
+            List.of(
+                new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23, (byte) 24 }),
+                new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 25, (byte) 26 })
+            )
         ).transformToCoordinationFormat();
 
         assertThat(
             results,
             is(
                 List.of(
-                    new org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingByteResults(
+                    new org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults(
                         TextEmbeddingByteResults.TEXT_EMBEDDING_BYTES,
-                        new byte[] { 23, 24 },
+                        new double[] { 23F, 24F },
                         false
                     ),
-                    new org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingByteResults(
+                    new org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults(
                         TextEmbeddingByteResults.TEXT_EMBEDDING_BYTES,
-                        new byte[] { 25, 26 },
+                        new double[] { 25F, 26F },
                         false
                     )
                 )
@@ -125,7 +129,7 @@ protected TextEmbeddingByteResults mutateInstance(TextEmbeddingByteResults insta
             int end = randomInt(instance.embeddings().size() - 1);
             return new TextEmbeddingByteResults(instance.embeddings().subList(0, end));
         } else {
-            List<ByteEmbedding> embeddings = new ArrayList<>(instance.embeddings());
+            List<TextEmbeddingByteResults.Embedding> embeddings = new ArrayList<>(instance.embeddings());
             embeddings.add(createRandomEmbedding());
             return new TextEmbeddingByteResults(embeddings);
         }
@@ -134,7 +138,7 @@ protected TextEmbeddingByteResults mutateInstance(TextEmbeddingByteResults insta
     public static Map<String, Object> buildExpectationByte(List<List<Byte>> embeddings) {
         return Map.of(
             TextEmbeddingByteResults.TEXT_EMBEDDING_BYTES,
-            embeddings.stream().map(embedding -> Map.of(ByteEmbedding.EMBEDDING, embedding)).toList()
+            embeddings.stream().map(embedding -> Map.of(TextEmbeddingByteResults.Embedding.EMBEDDING, embedding)).toList()
         );
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java
index bac25c9181dee..716568fdb5645 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java
@@ -10,8 +10,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.core.inference.results.ByteEmbedding;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 
@@ -25,7 +23,7 @@
 public class TextEmbeddingResultsTests extends AbstractWireSerializingTestCase<TextEmbeddingResults> {
     public static TextEmbeddingResults createRandomResults() {
         int embeddings = randomIntBetween(1, 10);
-        List<FloatEmbedding> embeddingResults = new ArrayList<>(embeddings);
+        List<TextEmbeddingResults.Embedding> embeddingResults = new ArrayList<>(embeddings);
 
         for (int i = 0; i < embeddings; i++) {
             embeddingResults.add(createRandomEmbedding());
@@ -34,18 +32,18 @@ public static TextEmbeddingResults createRandomResults() {
         return new TextEmbeddingResults(embeddingResults);
     }
 
-    private static FloatEmbedding createRandomEmbedding() {
+    private static TextEmbeddingResults.Embedding createRandomEmbedding() {
         int columns = randomIntBetween(1, 10);
         float[] floats = new float[columns];
         for (int i = 0; i < columns; i++) {
             floats[i] = randomFloat();
         }
 
-        return new FloatEmbedding(floats);
+        return new TextEmbeddingResults.Embedding(floats);
     }
 
     public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOException {
-        var entity = new TextEmbeddingResults(List.of(FloatEmbedding.of(List.of(0.1F))));
+        var entity = new TextEmbeddingResults(List.of(new TextEmbeddingResults.Embedding(new float[] { 0.1F })));
 
         String xContentResult = Strings.toString(entity, true, true);
         assertThat(xContentResult, is("""
@@ -61,7 +59,10 @@ public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOE
     }
 
     public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws IOException {
-        var entity = new TextEmbeddingResults(List.of(FloatEmbedding.of(List.of(0.1F)), FloatEmbedding.of(List.of(0.2F))));
+        var entity = new TextEmbeddingResults(
+            List.of(new TextEmbeddingResults.Embedding(new float[] { 0.1F }), new TextEmbeddingResults.Embedding(new float[] { 0.2F }))
+
+        );
 
         String xContentResult = Strings.toString(entity, true, true);
         assertThat(xContentResult, is("""
@@ -82,8 +83,12 @@ public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws I
     }
 
     public void testTransformToCoordinationFormat() {
-        var results = new TextEmbeddingResults(List.of(FloatEmbedding.of(List.of(0.1F, 0.2F)), FloatEmbedding.of(List.of(0.3F, 0.4F))))
-            .transformToCoordinationFormat();
+        var results = new TextEmbeddingResults(
+            List.of(
+                new TextEmbeddingResults.Embedding(new float[] { 0.1F, 0.2F }),
+                new TextEmbeddingResults.Embedding(new float[] { 0.3F, 0.4F })
+            )
+        ).transformToCoordinationFormat();
 
         assertThat(
             results,
@@ -122,18 +127,21 @@ protected TextEmbeddingResults mutateInstance(TextEmbeddingResults instance) thr
             int end = randomInt(instance.embeddings().size() - 1);
             return new TextEmbeddingResults(instance.embeddings().subList(0, end));
         } else {
-            List<FloatEmbedding> embeddings = new ArrayList<>(instance.embeddings());
+            List<TextEmbeddingResults.Embedding> embeddings = new ArrayList<>(instance.embeddings());
             embeddings.add(createRandomEmbedding());
             return new TextEmbeddingResults(embeddings);
         }
     }
 
     public static Map<String, Object> buildExpectationFloat(List<float[]> embeddings) {
-        return Map.of(TextEmbeddingResults.TEXT_EMBEDDING, embeddings.stream().map(FloatEmbedding::new).toList());
+        return Map.of(TextEmbeddingResults.TEXT_EMBEDDING, embeddings.stream().map(TextEmbeddingResults.Embedding::new).toList());
     }
 
     public static Map<String, Object> buildExpectationByte(List<byte[]> embeddings) {
-        return Map.of(TextEmbeddingByteResults.TEXT_EMBEDDING_BYTES, embeddings.stream().map(ByteEmbedding::new).toList());
+        return Map.of(
+            TextEmbeddingByteResults.TEXT_EMBEDDING_BYTES,
+            embeddings.stream().map(TextEmbeddingByteResults.Embedding::new).toList()
+        );
     }
 
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java
index edd9637d92dd8..44e3f34ffcc15 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java
@@ -777,7 +777,7 @@ public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingResults() {
 
         var size = listener.actionGet(TIMEOUT);
 
-        assertThat(size, is(textEmbedding.embeddings().get(0).getEmbedding().size()));
+        assertThat(size, is(textEmbedding.embeddings().get(0).getSize()));
     }
 
     public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingByteResults() {
@@ -801,7 +801,7 @@ public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingByteResults() {
 
         var size = listener.actionGet(TIMEOUT);
 
-        assertThat(size, is(textEmbedding.embeddings().get(0).getEmbedding().size()));
+        assertThat(size, is(textEmbedding.embeddings().get(0).getSize()));
     }
 
     private static <K, V> Map<K, V> modifiableMap(Map<K, V> aMap) {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
index 66097e62c3ed8..f06fee4b0b9c4 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingFloatResults;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests;
@@ -52,6 +51,7 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -1226,14 +1226,14 @@ public void testChunkedInfer_BatchesCalls() throws IOException {
                 var floatResult = (ChunkedTextEmbeddingFloatResults) results.get(0);
                 assertThat(floatResult.chunks(), hasSize(1));
                 assertEquals("foo", floatResult.chunks().get(0).matchedText());
-                assertEquals(FloatEmbedding.of(List.of(0.123f, -0.123f)), floatResult.chunks().get(0).embedding());
+                assertTrue(Arrays.equals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding()));
             }
             {
                 assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedTextEmbeddingFloatResults.class));
                 var floatResult = (ChunkedTextEmbeddingFloatResults) results.get(1);
                 assertThat(floatResult.chunks(), hasSize(1));
                 assertEquals("bar", floatResult.chunks().get(0).matchedText());
-                assertEquals(FloatEmbedding.of(List.of(0.223f, -0.223f)), floatResult.chunks().get(0).embedding());
+                assertTrue(Arrays.equals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding()));
             }
 
             MatcherAssert.assertThat(webServer.requests(), hasSize(1));
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
index 124b90e10dc88..cbac29c452772 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingFloatResults;
-import org.elasticsearch.xpack.core.inference.results.FloatEmbedding;
 import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests;
@@ -49,6 +48,7 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -1267,14 +1267,14 @@ public void testChunkedInfer_Batches() throws IOException {
                 var floatResult = (ChunkedTextEmbeddingFloatResults) results.get(0);
                 assertThat(floatResult.chunks(), hasSize(1));
                 assertEquals("foo", floatResult.chunks().get(0).matchedText());
-                assertEquals(FloatEmbedding.of(List.of(0.123f, -0.123f)), floatResult.chunks().get(0).embedding());
+                assertTrue(Arrays.equals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding()));
             }
             {
                 assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedTextEmbeddingFloatResults.class));
                 var floatResult = (ChunkedTextEmbeddingFloatResults) results.get(1);
                 assertThat(floatResult.chunks(), hasSize(1));
                 assertEquals("bar", floatResult.chunks().get(0).matchedText());
-                assertEquals(FloatEmbedding.of(List.of(0.223f, -0.223f)), floatResult.chunks().get(0).embedding());
+                assertTrue(Arrays.equals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding()));
             }
 
             assertThat(webServer.requests(), hasSize(1));

From a73f52500b7c63277907f801740c48034fcbf397 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 28 May 2024 12:38:36 -0400
Subject: [PATCH 012/208] ESQL: Remove `nested` shims (#109105)

This removes the shim layers for `nested` that we inherited from QL.
ESQL doesn't support nested yet and when it does support nested its not
going to use the same tricks that QL uses.
---
 .../esql/core/expression/FieldAttribute.java  | 20 -------------------
 .../core/planner/ExpressionTranslator.java    | 10 ----------
 .../core/planner/ExpressionTranslators.java   |  5 ++---
 .../esql/core/planner/TranslatorHandler.java  |  2 +-
 .../planner/EsqlExpressionTranslators.java    |  8 ++++----
 .../esql/planner/EsqlTranslatorHandler.java   |  3 +--
 6 files changed, 8 insertions(+), 40 deletions(-)

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
index fc63ab5d8e7f2..9ee9732b542a0 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
@@ -27,7 +26,6 @@
 public class FieldAttribute extends TypedAttribute {
 
     private final FieldAttribute parent;
-    private final FieldAttribute nestedParent;
     private final String path;
     private final EsField field;
 
@@ -67,16 +65,6 @@ public FieldAttribute(
         this.path = parent != null ? parent.name() : StringUtils.EMPTY;
         this.parent = parent;
         this.field = field;
-
-        // figure out the last nested parent
-        FieldAttribute nestedPar = null;
-        if (parent != null) {
-            nestedPar = parent.nestedParent;
-            if (parent.dataType() == DataTypes.NESTED) {
-                nestedPar = parent;
-            }
-        }
-        this.nestedParent = nestedPar;
     }
 
     @Override
@@ -97,14 +85,6 @@ public String qualifiedPath() {
         return qualifier() != null ? qualifier() + (Strings.hasText(path) ? "." + path : StringUtils.EMPTY) : path;
     }
 
-    public boolean isNested() {
-        return nestedParent != null;
-    }
-
-    public FieldAttribute nestedParent() {
-        return nestedParent;
-    }
-
     public EsField.Exact getExactInfo() {
         return field.getExactInfo();
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslator.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslator.java
index 2efbd7fbc077d..db148e2d63fa1 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslator.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslator.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.TypedAttribute;
-import org.elasticsearch.xpack.esql.core.querydsl.query.NestedQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.util.Check;
 import org.elasticsearch.xpack.esql.core.util.ReflectionUtils;
@@ -27,15 +26,6 @@ public Query translate(Expression exp, TranslatorHandler handler) {
 
     protected abstract Query asQuery(E e, TranslatorHandler handler);
 
-    public static Query wrapIfNested(Query query, Expression exp) {
-        if (query != null && exp instanceof FieldAttribute fa) {
-            if (fa.isNested()) {
-                return new NestedQuery(fa.source(), fa.nestedParent().name(), query);
-            }
-        }
-        return query;
-    }
-
     public static FieldAttribute checkIsFieldAttribute(Expression e) {
         Check.isTrue(e instanceof FieldAttribute, "Expected a FieldAttribute but received [{}]", e);
         return (FieldAttribute) e;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
index 39674a8a5a8b0..cf19b8722b258 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
@@ -105,7 +105,7 @@ public static Query doTranslate(RegexMatch e, TranslatorHandler handler) {
                 throw new QlIllegalArgumentException("Cannot translate query for " + e);
             }
 
-            return wrapIfNested(q, field);
+            return q;
         }
 
         private static Query translateField(RegexMatch e, String targetFieldName) {
@@ -182,10 +182,9 @@ protected Query asQuery(Not not, TranslatorHandler handler) {
         }
 
         public static Query doTranslate(Not not, TranslatorHandler handler) {
-            Expression e = not.field();
             Query wrappedQuery = handler.asQuery(not.field());
             Query q = wrappedQuery.negate(not.source());
-            return wrapIfNested(q, e);
+            return q;
         }
     }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/TranslatorHandler.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/TranslatorHandler.java
index 12c68ca425816..1ccbb04f7a69c 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/TranslatorHandler.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/TranslatorHandler.java
@@ -27,7 +27,7 @@ public interface TranslatorHandler {
 
     default Query wrapFunctionQuery(ScalarFunction sf, Expression field, Supplier<Query> querySupplier) {
         if (field instanceof FieldAttribute) {
-            return ExpressionTranslator.wrapIfNested(querySupplier.get(), field);
+            return querySupplier.get();
         }
         throw new QlIllegalArgumentException("Cannot translate expression:[" + sf.sourceText() + "]");
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
index 70aa39a6319db..d78d2d7f79d80 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
@@ -385,17 +385,17 @@ public static void checkSpatialRelatesFunction(Expression constantExpression, Sh
          * improvement overall.
          * TODO: Remove this method and call the parent method once the SingleValueQuery improvements have been made
          */
-        public static Query wrapFunctionQuery(Expression field, Supplier<Query> querySupplier) {
-            return ExpressionTranslator.wrapIfNested(querySupplier.get(), field);
+        public static Query wrapFunctionQuery(Supplier<Query> querySupplier) {
+            return querySupplier.get();
         }
 
         public static Query doTranslate(SpatialRelatesFunction bc, TranslatorHandler handler) {
             if (bc.left().foldable()) {
                 checkSpatialRelatesFunction(bc.left(), bc.queryRelation());
-                return wrapFunctionQuery(bc.right(), () -> translate(bc, handler, bc.right(), bc.left()));
+                return wrapFunctionQuery(() -> translate(bc, handler, bc.right(), bc.left()));
             } else {
                 checkSpatialRelatesFunction(bc.right(), bc.queryRelation());
-                return wrapFunctionQuery(bc.left(), () -> translate(bc, handler, bc.left(), bc.right()));
+                return wrapFunctionQuery(() -> translate(bc, handler, bc.left(), bc.right()));
             }
         }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlTranslatorHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlTranslatorHandler.java
index 3c4ae41117273..c07be82ed2a16 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlTranslatorHandler.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlTranslatorHandler.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull;
-import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator;
 import org.elasticsearch.xpack.esql.core.planner.TranslatorHandler;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -50,7 +49,7 @@ public Query wrapFunctionQuery(ScalarFunction sf, Expression field, Supplier<Que
             if ((sf instanceof IsNull || sf instanceof IsNotNull) == false) {
                 query = new SingleValueQuery(query, fa.name());
             }
-            return ExpressionTranslator.wrapIfNested(query, field);
+            return query;
         }
         if (field instanceof MetadataAttribute) {
             return querySupplier.get(); // MetadataAttributes are always single valued

From e4cb2c9f6dbf1c9bb88aa44be7dd7dc5c41b297b Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 28 May 2024 13:32:30 -0400
Subject: [PATCH 013/208] ESQL: Add parsing for a LOOKUP command (#109040)

This command will serve as a sort of "inline" enrich. This commit itself
is mostly antlr generated code and paranoid tests that the new `LOOKUP`
keyword doesn't clash with any variables named `lookup`.

I've also marked our ANTLR generated files as `linguist-generated` which
causes them to be hidden by default in github's UI. You can still click
a button to see them if you like. See
https://docs.github.com/en/repositories/working-with-files/managing-files/customizing-how-changed-files-appear-on-github
---
 .gitattributes                                |    5 +
 .../src/main/resources/lookup.csv-spec        |   52 +
 .../esql/src/main/antlr/EsqlBaseLexer.g4      |   45 +
 .../esql/src/main/antlr/EsqlBaseLexer.tokens  |  326 ++--
 .../esql/src/main/antlr/EsqlBaseParser.g4     |   13 +-
 .../esql/src/main/antlr/EsqlBaseParser.tokens |  326 ++--
 .../xpack/esql/action/EsqlCapabilities.java   |   15 +-
 .../xpack/esql/parser/EsqlBaseLexer.interp    |   34 +-
 .../xpack/esql/parser/EsqlBaseLexer.java      | 1730 +++++++++--------
 .../xpack/esql/parser/EsqlBaseParser.interp   |   18 +-
 .../xpack/esql/parser/EsqlBaseParser.java     | 1680 ++++++++--------
 .../parser/EsqlBaseParserBaseListener.java    |   24 +
 .../parser/EsqlBaseParserBaseVisitor.java     |   14 +
 .../esql/parser/EsqlBaseParserListener.java   |   20 +
 .../esql/parser/EsqlBaseParserVisitor.java    |   12 +
 .../xpack/esql/parser/ExpressionBuilder.java  |   44 +-
 .../xpack/esql/parser/LogicalPlanBuilder.java |   45 +-
 .../xpack/esql/analysis/AnalyzerTests.java    |    9 +
 .../rest-api-spec/test/esql/150_lookup.yml    |   42 +
 19 files changed, 2494 insertions(+), 1960 deletions(-)
 create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec
 create mode 100644 x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml

diff --git a/.gitattributes b/.gitattributes
index dfb3948ec7295..ebe1a34db3479 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -3,3 +3,8 @@ CHANGELOG.asciidoc  merge=union
 # These files contain expected text output, and should not be changed on
 # Windows
 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/*.asciidoc text eol=lf
+
+x-pack/plugin/esql/src/main/antlr/*.tokens linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*.interp linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer*.java linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser*.java linguist-generated=true
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec
new file mode 100644
index 0000000000000..f6dc28fa5db12
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec
@@ -0,0 +1,52 @@
+
+
+
+//
+// Make sure that the new LOOKUP syntax doesn't clash with any existing things
+// named "lookup"
+//
+rowNamedLookup
+ROW lookup = "a"
+;
+
+lookup:keyword
+             a
+;
+
+rowNamedLOOKUP
+ROW LOOKUP = "a"
+;
+
+LOOKUP:keyword
+             a
+;
+
+evalNamedLookup
+ROW a = "a" | EVAL lookup = CONCAT(a, "1")
+;
+
+a:keyword | lookup:keyword
+        a |             a1
+;
+
+dissectNamedLookup
+row a = "foo bar" | dissect a "foo %{lookup}";
+
+a:keyword | lookup:keyword
+  foo bar |            bar
+;
+
+renameIntoLookup
+row a = "foo bar" | RENAME a AS lookup;
+
+lookup:keyword
+       foo bar
+;
+
+sortOnLookup
+ROW lookup = "a" | SORT lookup
+;
+
+lookup:keyword
+             a
+;
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
index 3c4cf1a3383bd..fb424b16ac403 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
@@ -10,6 +10,7 @@ GROK : 'grok'                 -> pushMode(EXPRESSION_MODE);
 INLINESTATS : 'inlinestats'   -> pushMode(EXPRESSION_MODE);
 KEEP : 'keep'                 -> pushMode(PROJECT_MODE);
 LIMIT : 'limit'               -> pushMode(EXPRESSION_MODE);
+LOOKUP : 'lookup'             -> pushMode(LOOKUP_MODE);
 META : 'meta'                 -> pushMode(META_MODE);
 METRICS : 'metrics'           -> pushMode(METRICS_MODE);
 MV_EXPAND : 'mv_expand'       -> pushMode(MVEXPAND_MODE);
@@ -344,6 +345,50 @@ ENRICH_FIELD_WS
     : WS -> channel(HIDDEN)
     ;
 
+// LOOKUP ON key
+mode LOOKUP_MODE;
+LOOKUP_PIPE : PIPE -> type(PIPE), popMode;
+LOOKUP_COMMA : COMMA -> type(COMMA);
+LOOKUP_DOT: DOT -> type(DOT);
+LOOKUP_ON : ON -> type(ON), pushMode(LOOKUP_FIELD_MODE);
+
+LOOKUP_INDEX_UNQUOTED_IDENTIFIER
+    : INDEX_UNQUOTED_IDENTIFIER -> type(INDEX_UNQUOTED_IDENTIFIER)
+    ;
+
+LOOKUP_LINE_COMMENT
+    : LINE_COMMENT -> channel(HIDDEN)
+    ;
+
+LOOKUP_MULTILINE_COMMENT
+    : MULTILINE_COMMENT -> channel(HIDDEN)
+    ;
+
+LOOKUP_WS
+    : WS -> channel(HIDDEN)
+    ;
+
+mode LOOKUP_FIELD_MODE;
+LOOKUP_FIELD_PIPE : PIPE -> type(PIPE), popMode, popMode;
+LOOKUP_FIELD_COMMA : COMMA -> type(COMMA);
+LOOKUP_FIELD_DOT: DOT -> type(DOT);
+
+LOOKUP_FIELD_ID_PATTERN
+    : ID_PATTERN -> type(ID_PATTERN)
+    ;
+
+LOOKUP_FIELD_LINE_COMMENT
+    : LINE_COMMENT -> channel(HIDDEN)
+    ;
+
+LOOKUP_FIELD_MULTILINE_COMMENT
+    : MULTILINE_COMMENT -> channel(HIDDEN)
+    ;
+
+LOOKUP_FIELD_WS
+    : WS -> channel(HIDDEN)
+    ;
+
 mode MVEXPAND_MODE;
 MVEXPAND_PIPE : PIPE -> type(PIPE), popMode;
 MVEXPAND_DOT: DOT -> type(DOT);
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
index 0e2d796eda847..30ce0d5eea55b 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
@@ -8,112 +8,119 @@ GROK=7
 INLINESTATS=8
 KEEP=9
 LIMIT=10
-META=11
-METRICS=12
-MV_EXPAND=13
-RENAME=14
-ROW=15
-SHOW=16
-SORT=17
-STATS=18
-WHERE=19
-UNKNOWN_CMD=20
-LINE_COMMENT=21
-MULTILINE_COMMENT=22
-WS=23
-INDEX_UNQUOTED_IDENTIFIER=24
-EXPLAIN_WS=25
-EXPLAIN_LINE_COMMENT=26
-EXPLAIN_MULTILINE_COMMENT=27
-PIPE=28
-QUOTED_STRING=29
-INTEGER_LITERAL=30
-DECIMAL_LITERAL=31
-BY=32
-AND=33
-ASC=34
-ASSIGN=35
-CAST_OP=36
-COMMA=37
-DESC=38
-DOT=39
-FALSE=40
-FIRST=41
-LAST=42
-LP=43
-IN=44
-IS=45
-LIKE=46
-NOT=47
-NULL=48
-NULLS=49
-OR=50
-PARAM=51
-RLIKE=52
-RP=53
-TRUE=54
-EQ=55
-CIEQ=56
-NEQ=57
-LT=58
-LTE=59
-GT=60
-GTE=61
-PLUS=62
-MINUS=63
-ASTERISK=64
-SLASH=65
-PERCENT=66
-OPENING_BRACKET=67
-CLOSING_BRACKET=68
-UNQUOTED_IDENTIFIER=69
-QUOTED_IDENTIFIER=70
-EXPR_LINE_COMMENT=71
-EXPR_MULTILINE_COMMENT=72
-EXPR_WS=73
-METADATA=74
-FROM_LINE_COMMENT=75
-FROM_MULTILINE_COMMENT=76
-FROM_WS=77
-ID_PATTERN=78
-PROJECT_LINE_COMMENT=79
-PROJECT_MULTILINE_COMMENT=80
-PROJECT_WS=81
-AS=82
-RENAME_LINE_COMMENT=83
-RENAME_MULTILINE_COMMENT=84
-RENAME_WS=85
-ON=86
-WITH=87
-ENRICH_POLICY_NAME=88
-ENRICH_LINE_COMMENT=89
-ENRICH_MULTILINE_COMMENT=90
-ENRICH_WS=91
-ENRICH_FIELD_LINE_COMMENT=92
-ENRICH_FIELD_MULTILINE_COMMENT=93
-ENRICH_FIELD_WS=94
-MVEXPAND_LINE_COMMENT=95
-MVEXPAND_MULTILINE_COMMENT=96
-MVEXPAND_WS=97
-INFO=98
-SHOW_LINE_COMMENT=99
-SHOW_MULTILINE_COMMENT=100
-SHOW_WS=101
-FUNCTIONS=102
-META_LINE_COMMENT=103
-META_MULTILINE_COMMENT=104
-META_WS=105
-COLON=106
-SETTING=107
-SETTING_LINE_COMMENT=108
-SETTTING_MULTILINE_COMMENT=109
-SETTING_WS=110
-METRICS_LINE_COMMENT=111
-METRICS_MULTILINE_COMMENT=112
-METRICS_WS=113
-CLOSING_METRICS_LINE_COMMENT=114
-CLOSING_METRICS_MULTILINE_COMMENT=115
-CLOSING_METRICS_WS=116
+LOOKUP=11
+META=12
+METRICS=13
+MV_EXPAND=14
+RENAME=15
+ROW=16
+SHOW=17
+SORT=18
+STATS=19
+WHERE=20
+UNKNOWN_CMD=21
+LINE_COMMENT=22
+MULTILINE_COMMENT=23
+WS=24
+INDEX_UNQUOTED_IDENTIFIER=25
+EXPLAIN_WS=26
+EXPLAIN_LINE_COMMENT=27
+EXPLAIN_MULTILINE_COMMENT=28
+PIPE=29
+QUOTED_STRING=30
+INTEGER_LITERAL=31
+DECIMAL_LITERAL=32
+BY=33
+AND=34
+ASC=35
+ASSIGN=36
+CAST_OP=37
+COMMA=38
+DESC=39
+DOT=40
+FALSE=41
+FIRST=42
+LAST=43
+LP=44
+IN=45
+IS=46
+LIKE=47
+NOT=48
+NULL=49
+NULLS=50
+OR=51
+PARAM=52
+RLIKE=53
+RP=54
+TRUE=55
+EQ=56
+CIEQ=57
+NEQ=58
+LT=59
+LTE=60
+GT=61
+GTE=62
+PLUS=63
+MINUS=64
+ASTERISK=65
+SLASH=66
+PERCENT=67
+OPENING_BRACKET=68
+CLOSING_BRACKET=69
+UNQUOTED_IDENTIFIER=70
+QUOTED_IDENTIFIER=71
+EXPR_LINE_COMMENT=72
+EXPR_MULTILINE_COMMENT=73
+EXPR_WS=74
+METADATA=75
+FROM_LINE_COMMENT=76
+FROM_MULTILINE_COMMENT=77
+FROM_WS=78
+ID_PATTERN=79
+PROJECT_LINE_COMMENT=80
+PROJECT_MULTILINE_COMMENT=81
+PROJECT_WS=82
+AS=83
+RENAME_LINE_COMMENT=84
+RENAME_MULTILINE_COMMENT=85
+RENAME_WS=86
+ON=87
+WITH=88
+ENRICH_POLICY_NAME=89
+ENRICH_LINE_COMMENT=90
+ENRICH_MULTILINE_COMMENT=91
+ENRICH_WS=92
+ENRICH_FIELD_LINE_COMMENT=93
+ENRICH_FIELD_MULTILINE_COMMENT=94
+ENRICH_FIELD_WS=95
+LOOKUP_LINE_COMMENT=96
+LOOKUP_MULTILINE_COMMENT=97
+LOOKUP_WS=98
+LOOKUP_FIELD_LINE_COMMENT=99
+LOOKUP_FIELD_MULTILINE_COMMENT=100
+LOOKUP_FIELD_WS=101
+MVEXPAND_LINE_COMMENT=102
+MVEXPAND_MULTILINE_COMMENT=103
+MVEXPAND_WS=104
+INFO=105
+SHOW_LINE_COMMENT=106
+SHOW_MULTILINE_COMMENT=107
+SHOW_WS=108
+FUNCTIONS=109
+META_LINE_COMMENT=110
+META_MULTILINE_COMMENT=111
+META_WS=112
+COLON=113
+SETTING=114
+SETTING_LINE_COMMENT=115
+SETTTING_MULTILINE_COMMENT=116
+SETTING_WS=117
+METRICS_LINE_COMMENT=118
+METRICS_MULTILINE_COMMENT=119
+METRICS_WS=120
+CLOSING_METRICS_LINE_COMMENT=121
+CLOSING_METRICS_MULTILINE_COMMENT=122
+CLOSING_METRICS_WS=123
 'dissect'=1
 'drop'=2
 'enrich'=3
@@ -124,56 +131,57 @@ CLOSING_METRICS_WS=116
 'inlinestats'=8
 'keep'=9
 'limit'=10
-'meta'=11
-'metrics'=12
-'mv_expand'=13
-'rename'=14
-'row'=15
-'show'=16
-'sort'=17
-'stats'=18
-'where'=19
-'|'=28
-'by'=32
-'and'=33
-'asc'=34
-'='=35
-'::'=36
-','=37
-'desc'=38
-'.'=39
-'false'=40
-'first'=41
-'last'=42
-'('=43
-'in'=44
-'is'=45
-'like'=46
-'not'=47
-'null'=48
-'nulls'=49
-'or'=50
-'?'=51
-'rlike'=52
-')'=53
-'true'=54
-'=='=55
-'=~'=56
-'!='=57
-'<'=58
-'<='=59
-'>'=60
-'>='=61
-'+'=62
-'-'=63
-'*'=64
-'/'=65
-'%'=66
-']'=68
-'metadata'=74
-'as'=82
-'on'=86
-'with'=87
-'info'=98
-'functions'=102
-':'=106
+'lookup'=11
+'meta'=12
+'metrics'=13
+'mv_expand'=14
+'rename'=15
+'row'=16
+'show'=17
+'sort'=18
+'stats'=19
+'where'=20
+'|'=29
+'by'=33
+'and'=34
+'asc'=35
+'='=36
+'::'=37
+','=38
+'desc'=39
+'.'=40
+'false'=41
+'first'=42
+'last'=43
+'('=44
+'in'=45
+'is'=46
+'like'=47
+'not'=48
+'null'=49
+'nulls'=50
+'or'=51
+'?'=52
+'rlike'=53
+')'=54
+'true'=55
+'=='=56
+'=~'=57
+'!='=58
+'<'=59
+'<='=60
+'>'=61
+'>='=62
+'+'=63
+'-'=64
+'*'=65
+'/'=66
+'%'=67
+']'=69
+'metadata'=75
+'as'=83
+'on'=87
+'with'=88
+'info'=105
+'functions'=109
+':'=113
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
index 7064856567d3e..29892d464c0e5 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
@@ -32,6 +32,7 @@ processingCommand
     : evalCommand
     | inlinestatsCommand
     | limitCommand
+    | lookupCommand
     | keepCommand
     | sortCommand
     | statsCommand
@@ -150,6 +151,10 @@ qualifiedNamePattern
     : identifierPattern (DOT identifierPattern)*
     ;
 
+qualifiedNamePatterns
+    : qualifiedNamePattern (COMMA qualifiedNamePattern)*
+    ;
+
 identifier
     : UNQUOTED_IDENTIFIER
     | QUOTED_IDENTIFIER
@@ -185,11 +190,11 @@ orderExpression
     ;
 
 keepCommand
-    :  KEEP qualifiedNamePattern (COMMA qualifiedNamePattern)*
+    :  KEEP qualifiedNamePatterns
     ;
 
 dropCommand
-    : DROP qualifiedNamePattern (COMMA qualifiedNamePattern)*
+    : DROP qualifiedNamePatterns
     ;
 
 renameCommand
@@ -268,3 +273,7 @@ enrichCommand
 enrichWithClause
     : (newName=qualifiedNamePattern ASSIGN)? enrichField=qualifiedNamePattern
     ;
+
+lookupCommand
+    : LOOKUP tableName=INDEX_UNQUOTED_IDENTIFIER ON matchFields=qualifiedNamePatterns
+    ;
diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
index 0e2d796eda847..30ce0d5eea55b 100644
--- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
+++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
@@ -8,112 +8,119 @@ GROK=7
 INLINESTATS=8
 KEEP=9
 LIMIT=10
-META=11
-METRICS=12
-MV_EXPAND=13
-RENAME=14
-ROW=15
-SHOW=16
-SORT=17
-STATS=18
-WHERE=19
-UNKNOWN_CMD=20
-LINE_COMMENT=21
-MULTILINE_COMMENT=22
-WS=23
-INDEX_UNQUOTED_IDENTIFIER=24
-EXPLAIN_WS=25
-EXPLAIN_LINE_COMMENT=26
-EXPLAIN_MULTILINE_COMMENT=27
-PIPE=28
-QUOTED_STRING=29
-INTEGER_LITERAL=30
-DECIMAL_LITERAL=31
-BY=32
-AND=33
-ASC=34
-ASSIGN=35
-CAST_OP=36
-COMMA=37
-DESC=38
-DOT=39
-FALSE=40
-FIRST=41
-LAST=42
-LP=43
-IN=44
-IS=45
-LIKE=46
-NOT=47
-NULL=48
-NULLS=49
-OR=50
-PARAM=51
-RLIKE=52
-RP=53
-TRUE=54
-EQ=55
-CIEQ=56
-NEQ=57
-LT=58
-LTE=59
-GT=60
-GTE=61
-PLUS=62
-MINUS=63
-ASTERISK=64
-SLASH=65
-PERCENT=66
-OPENING_BRACKET=67
-CLOSING_BRACKET=68
-UNQUOTED_IDENTIFIER=69
-QUOTED_IDENTIFIER=70
-EXPR_LINE_COMMENT=71
-EXPR_MULTILINE_COMMENT=72
-EXPR_WS=73
-METADATA=74
-FROM_LINE_COMMENT=75
-FROM_MULTILINE_COMMENT=76
-FROM_WS=77
-ID_PATTERN=78
-PROJECT_LINE_COMMENT=79
-PROJECT_MULTILINE_COMMENT=80
-PROJECT_WS=81
-AS=82
-RENAME_LINE_COMMENT=83
-RENAME_MULTILINE_COMMENT=84
-RENAME_WS=85
-ON=86
-WITH=87
-ENRICH_POLICY_NAME=88
-ENRICH_LINE_COMMENT=89
-ENRICH_MULTILINE_COMMENT=90
-ENRICH_WS=91
-ENRICH_FIELD_LINE_COMMENT=92
-ENRICH_FIELD_MULTILINE_COMMENT=93
-ENRICH_FIELD_WS=94
-MVEXPAND_LINE_COMMENT=95
-MVEXPAND_MULTILINE_COMMENT=96
-MVEXPAND_WS=97
-INFO=98
-SHOW_LINE_COMMENT=99
-SHOW_MULTILINE_COMMENT=100
-SHOW_WS=101
-FUNCTIONS=102
-META_LINE_COMMENT=103
-META_MULTILINE_COMMENT=104
-META_WS=105
-COLON=106
-SETTING=107
-SETTING_LINE_COMMENT=108
-SETTTING_MULTILINE_COMMENT=109
-SETTING_WS=110
-METRICS_LINE_COMMENT=111
-METRICS_MULTILINE_COMMENT=112
-METRICS_WS=113
-CLOSING_METRICS_LINE_COMMENT=114
-CLOSING_METRICS_MULTILINE_COMMENT=115
-CLOSING_METRICS_WS=116
+LOOKUP=11
+META=12
+METRICS=13
+MV_EXPAND=14
+RENAME=15
+ROW=16
+SHOW=17
+SORT=18
+STATS=19
+WHERE=20
+UNKNOWN_CMD=21
+LINE_COMMENT=22
+MULTILINE_COMMENT=23
+WS=24
+INDEX_UNQUOTED_IDENTIFIER=25
+EXPLAIN_WS=26
+EXPLAIN_LINE_COMMENT=27
+EXPLAIN_MULTILINE_COMMENT=28
+PIPE=29
+QUOTED_STRING=30
+INTEGER_LITERAL=31
+DECIMAL_LITERAL=32
+BY=33
+AND=34
+ASC=35
+ASSIGN=36
+CAST_OP=37
+COMMA=38
+DESC=39
+DOT=40
+FALSE=41
+FIRST=42
+LAST=43
+LP=44
+IN=45
+IS=46
+LIKE=47
+NOT=48
+NULL=49
+NULLS=50
+OR=51
+PARAM=52
+RLIKE=53
+RP=54
+TRUE=55
+EQ=56
+CIEQ=57
+NEQ=58
+LT=59
+LTE=60
+GT=61
+GTE=62
+PLUS=63
+MINUS=64
+ASTERISK=65
+SLASH=66
+PERCENT=67
+OPENING_BRACKET=68
+CLOSING_BRACKET=69
+UNQUOTED_IDENTIFIER=70
+QUOTED_IDENTIFIER=71
+EXPR_LINE_COMMENT=72
+EXPR_MULTILINE_COMMENT=73
+EXPR_WS=74
+METADATA=75
+FROM_LINE_COMMENT=76
+FROM_MULTILINE_COMMENT=77
+FROM_WS=78
+ID_PATTERN=79
+PROJECT_LINE_COMMENT=80
+PROJECT_MULTILINE_COMMENT=81
+PROJECT_WS=82
+AS=83
+RENAME_LINE_COMMENT=84
+RENAME_MULTILINE_COMMENT=85
+RENAME_WS=86
+ON=87
+WITH=88
+ENRICH_POLICY_NAME=89
+ENRICH_LINE_COMMENT=90
+ENRICH_MULTILINE_COMMENT=91
+ENRICH_WS=92
+ENRICH_FIELD_LINE_COMMENT=93
+ENRICH_FIELD_MULTILINE_COMMENT=94
+ENRICH_FIELD_WS=95
+LOOKUP_LINE_COMMENT=96
+LOOKUP_MULTILINE_COMMENT=97
+LOOKUP_WS=98
+LOOKUP_FIELD_LINE_COMMENT=99
+LOOKUP_FIELD_MULTILINE_COMMENT=100
+LOOKUP_FIELD_WS=101
+MVEXPAND_LINE_COMMENT=102
+MVEXPAND_MULTILINE_COMMENT=103
+MVEXPAND_WS=104
+INFO=105
+SHOW_LINE_COMMENT=106
+SHOW_MULTILINE_COMMENT=107
+SHOW_WS=108
+FUNCTIONS=109
+META_LINE_COMMENT=110
+META_MULTILINE_COMMENT=111
+META_WS=112
+COLON=113
+SETTING=114
+SETTING_LINE_COMMENT=115
+SETTTING_MULTILINE_COMMENT=116
+SETTING_WS=117
+METRICS_LINE_COMMENT=118
+METRICS_MULTILINE_COMMENT=119
+METRICS_WS=120
+CLOSING_METRICS_LINE_COMMENT=121
+CLOSING_METRICS_MULTILINE_COMMENT=122
+CLOSING_METRICS_WS=123
 'dissect'=1
 'drop'=2
 'enrich'=3
@@ -124,56 +131,57 @@ CLOSING_METRICS_WS=116
 'inlinestats'=8
 'keep'=9
 'limit'=10
-'meta'=11
-'metrics'=12
-'mv_expand'=13
-'rename'=14
-'row'=15
-'show'=16
-'sort'=17
-'stats'=18
-'where'=19
-'|'=28
-'by'=32
-'and'=33
-'asc'=34
-'='=35
-'::'=36
-','=37
-'desc'=38
-'.'=39
-'false'=40
-'first'=41
-'last'=42
-'('=43
-'in'=44
-'is'=45
-'like'=46
-'not'=47
-'null'=48
-'nulls'=49
-'or'=50
-'?'=51
-'rlike'=52
-')'=53
-'true'=54
-'=='=55
-'=~'=56
-'!='=57
-'<'=58
-'<='=59
-'>'=60
-'>='=61
-'+'=62
-'-'=63
-'*'=64
-'/'=65
-'%'=66
-']'=68
-'metadata'=74
-'as'=82
-'on'=86
-'with'=87
-'info'=98
-'functions'=102
-':'=106
+'lookup'=11
+'meta'=12
+'metrics'=13
+'mv_expand'=14
+'rename'=15
+'row'=16
+'show'=17
+'sort'=18
+'stats'=19
+'where'=20
+'|'=29
+'by'=33
+'and'=34
+'asc'=35
+'='=36
+'::'=37
+','=38
+'desc'=39
+'.'=40
+'false'=41
+'first'=42
+'last'=43
+'('=44
+'in'=45
+'is'=46
+'like'=47
+'not'=48
+'null'=49
+'nulls'=50
+'or'=51
+'?'=52
+'rlike'=53
+')'=54
+'true'=55
+'=='=56
+'=~'=57
+'!='=58
+'<'=59
+'<='=60
+'>'=61
+'>='=62
+'+'=63
+'-'=64
+'*'=65
+'/'=66
+'%'=67
+']'=69
+'metadata'=75
+'as'=83
+'on'=87
+'with'=88
+'info'=105
+'functions'=109
+':'=113
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index a6f7cc2317e11..ca90b9e1e3f2e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.action;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction;
 import org.elasticsearch.xpack.esql.plugin.EsqlFeatures;
@@ -36,10 +37,22 @@ public class EsqlCapabilities {
      */
     private static final String METADATA_IGNORED_FIELD = "metadata_field_ignored";
 
+    /**
+     * Support for requesting the "LOOKUP" command.
+     */
+    private static final String LOOKUP = "lookup";
+
     public static final Set<String> CAPABILITIES = capabilities();
 
     private static Set<String> capabilities() {
-        List<String> caps = new ArrayList<>(List.of(FN_CBRT, ST_CENTROID_AGG_OPTIMIZED, METADATA_IGNORED_FIELD));
+        List<String> caps = new ArrayList<>();
+        caps.add(FN_CBRT);
+        caps.add(ST_CENTROID_AGG_OPTIMIZED);
+        caps.add(METADATA_IGNORED_FIELD);
+
+        if (Build.current().isSnapshot()) {
+            caps.add(LOOKUP);
+        }
 
         /*
          * Add all of our cluster features without the leading "esql."
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
index 38693348204af..6f9e20e95430b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
@@ -10,6 +10,7 @@ null
 'inlinestats'
 'keep'
 'limit'
+'lookup'
 'meta'
 'metrics'
 'mv_expand'
@@ -97,6 +98,12 @@ null
 null
 null
 null
+null
+null
+null
+null
+null
+null
 'info'
 null
 null
@@ -129,6 +136,7 @@ GROK
 INLINESTATS
 KEEP
 LIMIT
+LOOKUP
 META
 METRICS
 MV_EXPAND
@@ -213,6 +221,12 @@ ENRICH_WS
 ENRICH_FIELD_LINE_COMMENT
 ENRICH_FIELD_MULTILINE_COMMENT
 ENRICH_FIELD_WS
+LOOKUP_LINE_COMMENT
+LOOKUP_MULTILINE_COMMENT
+LOOKUP_WS
+LOOKUP_FIELD_LINE_COMMENT
+LOOKUP_FIELD_MULTILINE_COMMENT
+LOOKUP_FIELD_WS
 MVEXPAND_LINE_COMMENT
 MVEXPAND_MULTILINE_COMMENT
 MVEXPAND_WS
@@ -247,6 +261,7 @@ GROK
 INLINESTATS
 KEEP
 LIMIT
+LOOKUP
 META
 METRICS
 MV_EXPAND
@@ -374,6 +389,21 @@ ENRICH_FIELD_QUOTED_IDENTIFIER
 ENRICH_FIELD_LINE_COMMENT
 ENRICH_FIELD_MULTILINE_COMMENT
 ENRICH_FIELD_WS
+LOOKUP_PIPE
+LOOKUP_COMMA
+LOOKUP_DOT
+LOOKUP_ON
+LOOKUP_INDEX_UNQUOTED_IDENTIFIER
+LOOKUP_LINE_COMMENT
+LOOKUP_MULTILINE_COMMENT
+LOOKUP_WS
+LOOKUP_FIELD_PIPE
+LOOKUP_FIELD_COMMA
+LOOKUP_FIELD_DOT
+LOOKUP_FIELD_ID_PATTERN
+LOOKUP_FIELD_LINE_COMMENT
+LOOKUP_FIELD_MULTILINE_COMMENT
+LOOKUP_FIELD_WS
 MVEXPAND_PIPE
 MVEXPAND_DOT
 MVEXPAND_QUOTED_IDENTIFIER
@@ -424,6 +454,8 @@ PROJECT_MODE
 RENAME_MODE
 ENRICH_MODE
 ENRICH_FIELD_MODE
+LOOKUP_MODE
+LOOKUP_FIELD_MODE
 MVEXPAND_MODE
 SHOW_MODE
 META_MODE
@@ -432,4 +464,4 @@ METRICS_MODE
 CLOSING_METRICS_MODE
 
 atn:
-[4, 0, 116, 1297, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 4, 19, 522, 8, 19, 11, 19, 12, 19, 523, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 532, 8, 20, 10, 20, 12, 20, 535, 9, 20, 1, 20, 3, 20, 538, 8, 20, 1, 20, 3, 20, 541, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 550, 8, 21, 10, 21, 12, 21, 553, 9, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 4, 22, 561, 8, 22, 11, 22, 12, 22, 562, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 3, 23, 570, 8, 23, 1, 24, 4, 24, 573, 8, 24, 11, 24, 12, 24, 574, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 28, 1, 29, 1, 29, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 35, 1, 35, 3, 35, 614, 8, 35, 1, 35, 4, 35, 617, 8, 35, 11, 35, 12, 35, 618, 1, 36, 1, 36, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 3, 38, 628, 8, 38, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 3, 40, 635, 8, 40, 1, 41, 1, 41, 1, 41, 5, 41, 640, 8, 41, 10, 41, 12, 41, 643, 9, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 5, 41, 651, 8, 41, 10, 41, 12, 41, 654, 9, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 3, 41, 661, 8, 41, 1, 41, 3, 41, 664, 8, 41, 3, 41, 666, 8, 41, 1, 42, 4, 42, 669, 8, 42, 11, 42, 12, 42, 670, 1, 43, 4, 43, 674, 8, 43, 11, 43, 12, 43, 675, 1, 43, 1, 43, 5, 43, 680, 8, 43, 10, 43, 12, 43, 683, 9, 43, 1, 43, 1, 43, 4, 43, 687, 8, 43, 11, 43, 12, 43, 688, 1, 43, 4, 43, 692, 8, 43, 11, 43, 12, 43, 693, 1, 43, 1, 43, 5, 43, 698, 8, 43, 10, 43, 12, 43, 701, 9, 43, 3, 43, 703, 8, 43, 1, 43, 1, 43, 1, 43, 1, 43, 4, 43, 709, 8, 43, 11, 43, 12, 43, 710, 1, 43, 1, 43, 3, 43, 715, 8, 43, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 74, 1, 74, 1, 75, 1, 75, 1, 76, 1, 76, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 79, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 5, 81, 846, 8, 81, 10, 81, 12, 81, 849, 9, 81, 1, 81, 1, 81, 3, 81, 853, 8, 81, 1, 81, 4, 81, 856, 8, 81, 11, 81, 12, 81, 857, 3, 81, 860, 8, 81, 1, 82, 1, 82, 4, 82, 864, 8, 82, 11, 82, 12, 82, 865, 1, 82, 1, 82, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 3, 101, 951, 8, 101, 1, 102, 1, 102, 3, 102, 955, 8, 102, 1, 102, 5, 102, 958, 8, 102, 10, 102, 12, 102, 961, 9, 102, 1, 102, 1, 102, 3, 102, 965, 8, 102, 1, 102, 4, 102, 968, 8, 102, 11, 102, 12, 102, 969, 3, 102, 972, 8, 102, 1, 103, 1, 103, 4, 103, 976, 8, 103, 11, 103, 12, 103, 977, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 121, 4, 121, 1053, 8, 121, 11, 121, 12, 121, 1054, 1, 121, 1, 121, 3, 121, 1059, 8, 121, 1, 121, 4, 121, 1062, 8, 121, 11, 121, 12, 121, 1063, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 150, 1, 150, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 156, 4, 156, 1218, 8, 156, 11, 156, 12, 156, 1219, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 172, 2, 551, 652, 0, 173, 14, 1, 16, 2, 18, 3, 20, 4, 22, 5, 24, 6, 26, 7, 28, 8, 30, 9, 32, 10, 34, 11, 36, 12, 38, 13, 40, 14, 42, 15, 44, 16, 46, 17, 48, 18, 50, 19, 52, 20, 54, 21, 56, 22, 58, 23, 60, 0, 62, 24, 64, 0, 66, 0, 68, 25, 70, 26, 72, 27, 74, 28, 76, 0, 78, 0, 80, 0, 82, 0, 84, 0, 86, 0, 88, 0, 90, 0, 92, 0, 94, 0, 96, 29, 98, 30, 100, 31, 102, 32, 104, 33, 106, 34, 108, 35, 110, 36, 112, 37, 114, 38, 116, 39, 118, 40, 120, 41, 122, 42, 124, 43, 126, 44, 128, 45, 130, 46, 132, 47, 134, 48, 136, 49, 138, 50, 140, 51, 142, 52, 144, 53, 146, 54, 148, 55, 150, 56, 152, 57, 154, 58, 156, 59, 158, 60, 160, 61, 162, 62, 164, 63, 166, 64, 168, 65, 170, 66, 172, 67, 174, 68, 176, 69, 178, 0, 180, 70, 182, 71, 184, 72, 186, 73, 188, 0, 190, 0, 192, 0, 194, 0, 196, 0, 198, 0, 200, 74, 202, 0, 204, 75, 206, 76, 208, 77, 210, 0, 212, 0, 214, 0, 216, 0, 218, 0, 220, 78, 222, 79, 224, 80, 226, 81, 228, 0, 230, 0, 232, 0, 234, 0, 236, 82, 238, 0, 240, 83, 242, 84, 244, 85, 246, 0, 248, 0, 250, 86, 252, 87, 254, 0, 256, 88, 258, 0, 260, 0, 262, 89, 264, 90, 266, 91, 268, 0, 270, 0, 272, 0, 274, 0, 276, 0, 278, 0, 280, 0, 282, 92, 284, 93, 286, 94, 288, 0, 290, 0, 292, 0, 294, 0, 296, 95, 298, 96, 300, 97, 302, 0, 304, 98, 306, 99, 308, 100, 310, 101, 312, 0, 314, 102, 316, 103, 318, 104, 320, 105, 322, 0, 324, 106, 326, 107, 328, 108, 330, 109, 332, 110, 334, 0, 336, 0, 338, 111, 340, 112, 342, 113, 344, 0, 346, 114, 348, 115, 350, 116, 352, 0, 354, 0, 356, 0, 358, 0, 14, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 13, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 10, 0, 9, 10, 13, 13, 32, 32, 44, 44, 47, 47, 61, 61, 91, 91, 93, 93, 96, 96, 124, 124, 2, 0, 42, 42, 47, 47, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 5, 0, 34, 34, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 69, 69, 101, 101, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1322, 0, 14, 1, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 52, 1, 0, 0, 0, 0, 54, 1, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 58, 1, 0, 0, 0, 0, 62, 1, 0, 0, 0, 1, 64, 1, 0, 0, 0, 1, 66, 1, 0, 0, 0, 1, 68, 1, 0, 0, 0, 1, 70, 1, 0, 0, 0, 1, 72, 1, 0, 0, 0, 2, 74, 1, 0, 0, 0, 2, 96, 1, 0, 0, 0, 2, 98, 1, 0, 0, 0, 2, 100, 1, 0, 0, 0, 2, 102, 1, 0, 0, 0, 2, 104, 1, 0, 0, 0, 2, 106, 1, 0, 0, 0, 2, 108, 1, 0, 0, 0, 2, 110, 1, 0, 0, 0, 2, 112, 1, 0, 0, 0, 2, 114, 1, 0, 0, 0, 2, 116, 1, 0, 0, 0, 2, 118, 1, 0, 0, 0, 2, 120, 1, 0, 0, 0, 2, 122, 1, 0, 0, 0, 2, 124, 1, 0, 0, 0, 2, 126, 1, 0, 0, 0, 2, 128, 1, 0, 0, 0, 2, 130, 1, 0, 0, 0, 2, 132, 1, 0, 0, 0, 2, 134, 1, 0, 0, 0, 2, 136, 1, 0, 0, 0, 2, 138, 1, 0, 0, 0, 2, 140, 1, 0, 0, 0, 2, 142, 1, 0, 0, 0, 2, 144, 1, 0, 0, 0, 2, 146, 1, 0, 0, 0, 2, 148, 1, 0, 0, 0, 2, 150, 1, 0, 0, 0, 2, 152, 1, 0, 0, 0, 2, 154, 1, 0, 0, 0, 2, 156, 1, 0, 0, 0, 2, 158, 1, 0, 0, 0, 2, 160, 1, 0, 0, 0, 2, 162, 1, 0, 0, 0, 2, 164, 1, 0, 0, 0, 2, 166, 1, 0, 0, 0, 2, 168, 1, 0, 0, 0, 2, 170, 1, 0, 0, 0, 2, 172, 1, 0, 0, 0, 2, 174, 1, 0, 0, 0, 2, 176, 1, 0, 0, 0, 2, 180, 1, 0, 0, 0, 2, 182, 1, 0, 0, 0, 2, 184, 1, 0, 0, 0, 2, 186, 1, 0, 0, 0, 3, 188, 1, 0, 0, 0, 3, 190, 1, 0, 0, 0, 3, 192, 1, 0, 0, 0, 3, 194, 1, 0, 0, 0, 3, 196, 1, 0, 0, 0, 3, 198, 1, 0, 0, 0, 3, 200, 1, 0, 0, 0, 3, 202, 1, 0, 0, 0, 3, 204, 1, 0, 0, 0, 3, 206, 1, 0, 0, 0, 3, 208, 1, 0, 0, 0, 4, 210, 1, 0, 0, 0, 4, 212, 1, 0, 0, 0, 4, 214, 1, 0, 0, 0, 4, 220, 1, 0, 0, 0, 4, 222, 1, 0, 0, 0, 4, 224, 1, 0, 0, 0, 4, 226, 1, 0, 0, 0, 5, 228, 1, 0, 0, 0, 5, 230, 1, 0, 0, 0, 5, 232, 1, 0, 0, 0, 5, 234, 1, 0, 0, 0, 5, 236, 1, 0, 0, 0, 5, 238, 1, 0, 0, 0, 5, 240, 1, 0, 0, 0, 5, 242, 1, 0, 0, 0, 5, 244, 1, 0, 0, 0, 6, 246, 1, 0, 0, 0, 6, 248, 1, 0, 0, 0, 6, 250, 1, 0, 0, 0, 6, 252, 1, 0, 0, 0, 6, 256, 1, 0, 0, 0, 6, 258, 1, 0, 0, 0, 6, 260, 1, 0, 0, 0, 6, 262, 1, 0, 0, 0, 6, 264, 1, 0, 0, 0, 6, 266, 1, 0, 0, 0, 7, 268, 1, 0, 0, 0, 7, 270, 1, 0, 0, 0, 7, 272, 1, 0, 0, 0, 7, 274, 1, 0, 0, 0, 7, 276, 1, 0, 0, 0, 7, 278, 1, 0, 0, 0, 7, 280, 1, 0, 0, 0, 7, 282, 1, 0, 0, 0, 7, 284, 1, 0, 0, 0, 7, 286, 1, 0, 0, 0, 8, 288, 1, 0, 0, 0, 8, 290, 1, 0, 0, 0, 8, 292, 1, 0, 0, 0, 8, 294, 1, 0, 0, 0, 8, 296, 1, 0, 0, 0, 8, 298, 1, 0, 0, 0, 8, 300, 1, 0, 0, 0, 9, 302, 1, 0, 0, 0, 9, 304, 1, 0, 0, 0, 9, 306, 1, 0, 0, 0, 9, 308, 1, 0, 0, 0, 9, 310, 1, 0, 0, 0, 10, 312, 1, 0, 0, 0, 10, 314, 1, 0, 0, 0, 10, 316, 1, 0, 0, 0, 10, 318, 1, 0, 0, 0, 10, 320, 1, 0, 0, 0, 11, 322, 1, 0, 0, 0, 11, 324, 1, 0, 0, 0, 11, 326, 1, 0, 0, 0, 11, 328, 1, 0, 0, 0, 11, 330, 1, 0, 0, 0, 11, 332, 1, 0, 0, 0, 12, 334, 1, 0, 0, 0, 12, 336, 1, 0, 0, 0, 12, 338, 1, 0, 0, 0, 12, 340, 1, 0, 0, 0, 12, 342, 1, 0, 0, 0, 13, 344, 1, 0, 0, 0, 13, 346, 1, 0, 0, 0, 13, 348, 1, 0, 0, 0, 13, 350, 1, 0, 0, 0, 13, 352, 1, 0, 0, 0, 13, 354, 1, 0, 0, 0, 13, 356, 1, 0, 0, 0, 13, 358, 1, 0, 0, 0, 14, 360, 1, 0, 0, 0, 16, 370, 1, 0, 0, 0, 18, 377, 1, 0, 0, 0, 20, 386, 1, 0, 0, 0, 22, 393, 1, 0, 0, 0, 24, 403, 1, 0, 0, 0, 26, 410, 1, 0, 0, 0, 28, 417, 1, 0, 0, 0, 30, 431, 1, 0, 0, 0, 32, 438, 1, 0, 0, 0, 34, 446, 1, 0, 0, 0, 36, 453, 1, 0, 0, 0, 38, 463, 1, 0, 0, 0, 40, 475, 1, 0, 0, 0, 42, 484, 1, 0, 0, 0, 44, 490, 1, 0, 0, 0, 46, 497, 1, 0, 0, 0, 48, 504, 1, 0, 0, 0, 50, 512, 1, 0, 0, 0, 52, 521, 1, 0, 0, 0, 54, 527, 1, 0, 0, 0, 56, 544, 1, 0, 0, 0, 58, 560, 1, 0, 0, 0, 60, 569, 1, 0, 0, 0, 62, 572, 1, 0, 0, 0, 64, 576, 1, 0, 0, 0, 66, 581, 1, 0, 0, 0, 68, 586, 1, 0, 0, 0, 70, 590, 1, 0, 0, 0, 72, 594, 1, 0, 0, 0, 74, 598, 1, 0, 0, 0, 76, 602, 1, 0, 0, 0, 78, 604, 1, 0, 0, 0, 80, 606, 1, 0, 0, 0, 82, 609, 1, 0, 0, 0, 84, 611, 1, 0, 0, 0, 86, 620, 1, 0, 0, 0, 88, 622, 1, 0, 0, 0, 90, 627, 1, 0, 0, 0, 92, 629, 1, 0, 0, 0, 94, 634, 1, 0, 0, 0, 96, 665, 1, 0, 0, 0, 98, 668, 1, 0, 0, 0, 100, 714, 1, 0, 0, 0, 102, 716, 1, 0, 0, 0, 104, 719, 1, 0, 0, 0, 106, 723, 1, 0, 0, 0, 108, 727, 1, 0, 0, 0, 110, 729, 1, 0, 0, 0, 112, 732, 1, 0, 0, 0, 114, 734, 1, 0, 0, 0, 116, 739, 1, 0, 0, 0, 118, 741, 1, 0, 0, 0, 120, 747, 1, 0, 0, 0, 122, 753, 1, 0, 0, 0, 124, 758, 1, 0, 0, 0, 126, 760, 1, 0, 0, 0, 128, 763, 1, 0, 0, 0, 130, 766, 1, 0, 0, 0, 132, 771, 1, 0, 0, 0, 134, 775, 1, 0, 0, 0, 136, 780, 1, 0, 0, 0, 138, 786, 1, 0, 0, 0, 140, 789, 1, 0, 0, 0, 142, 791, 1, 0, 0, 0, 144, 797, 1, 0, 0, 0, 146, 799, 1, 0, 0, 0, 148, 804, 1, 0, 0, 0, 150, 807, 1, 0, 0, 0, 152, 810, 1, 0, 0, 0, 154, 813, 1, 0, 0, 0, 156, 815, 1, 0, 0, 0, 158, 818, 1, 0, 0, 0, 160, 820, 1, 0, 0, 0, 162, 823, 1, 0, 0, 0, 164, 825, 1, 0, 0, 0, 166, 827, 1, 0, 0, 0, 168, 829, 1, 0, 0, 0, 170, 831, 1, 0, 0, 0, 172, 833, 1, 0, 0, 0, 174, 838, 1, 0, 0, 0, 176, 859, 1, 0, 0, 0, 178, 861, 1, 0, 0, 0, 180, 869, 1, 0, 0, 0, 182, 871, 1, 0, 0, 0, 184, 875, 1, 0, 0, 0, 186, 879, 1, 0, 0, 0, 188, 883, 1, 0, 0, 0, 190, 888, 1, 0, 0, 0, 192, 892, 1, 0, 0, 0, 194, 896, 1, 0, 0, 0, 196, 900, 1, 0, 0, 0, 198, 904, 1, 0, 0, 0, 200, 908, 1, 0, 0, 0, 202, 917, 1, 0, 0, 0, 204, 921, 1, 0, 0, 0, 206, 925, 1, 0, 0, 0, 208, 929, 1, 0, 0, 0, 210, 933, 1, 0, 0, 0, 212, 938, 1, 0, 0, 0, 214, 942, 1, 0, 0, 0, 216, 950, 1, 0, 0, 0, 218, 971, 1, 0, 0, 0, 220, 975, 1, 0, 0, 0, 222, 979, 1, 0, 0, 0, 224, 983, 1, 0, 0, 0, 226, 987, 1, 0, 0, 0, 228, 991, 1, 0, 0, 0, 230, 996, 1, 0, 0, 0, 232, 1000, 1, 0, 0, 0, 234, 1004, 1, 0, 0, 0, 236, 1008, 1, 0, 0, 0, 238, 1011, 1, 0, 0, 0, 240, 1015, 1, 0, 0, 0, 242, 1019, 1, 0, 0, 0, 244, 1023, 1, 0, 0, 0, 246, 1027, 1, 0, 0, 0, 248, 1032, 1, 0, 0, 0, 250, 1037, 1, 0, 0, 0, 252, 1042, 1, 0, 0, 0, 254, 1049, 1, 0, 0, 0, 256, 1058, 1, 0, 0, 0, 258, 1065, 1, 0, 0, 0, 260, 1069, 1, 0, 0, 0, 262, 1073, 1, 0, 0, 0, 264, 1077, 1, 0, 0, 0, 266, 1081, 1, 0, 0, 0, 268, 1085, 1, 0, 0, 0, 270, 1091, 1, 0, 0, 0, 272, 1095, 1, 0, 0, 0, 274, 1099, 1, 0, 0, 0, 276, 1103, 1, 0, 0, 0, 278, 1107, 1, 0, 0, 0, 280, 1111, 1, 0, 0, 0, 282, 1115, 1, 0, 0, 0, 284, 1119, 1, 0, 0, 0, 286, 1123, 1, 0, 0, 0, 288, 1127, 1, 0, 0, 0, 290, 1132, 1, 0, 0, 0, 292, 1136, 1, 0, 0, 0, 294, 1140, 1, 0, 0, 0, 296, 1144, 1, 0, 0, 0, 298, 1148, 1, 0, 0, 0, 300, 1152, 1, 0, 0, 0, 302, 1156, 1, 0, 0, 0, 304, 1161, 1, 0, 0, 0, 306, 1166, 1, 0, 0, 0, 308, 1170, 1, 0, 0, 0, 310, 1174, 1, 0, 0, 0, 312, 1178, 1, 0, 0, 0, 314, 1183, 1, 0, 0, 0, 316, 1193, 1, 0, 0, 0, 318, 1197, 1, 0, 0, 0, 320, 1201, 1, 0, 0, 0, 322, 1205, 1, 0, 0, 0, 324, 1210, 1, 0, 0, 0, 326, 1217, 1, 0, 0, 0, 328, 1221, 1, 0, 0, 0, 330, 1225, 1, 0, 0, 0, 332, 1229, 1, 0, 0, 0, 334, 1233, 1, 0, 0, 0, 336, 1238, 1, 0, 0, 0, 338, 1244, 1, 0, 0, 0, 340, 1248, 1, 0, 0, 0, 342, 1252, 1, 0, 0, 0, 344, 1256, 1, 0, 0, 0, 346, 1262, 1, 0, 0, 0, 348, 1266, 1, 0, 0, 0, 350, 1270, 1, 0, 0, 0, 352, 1274, 1, 0, 0, 0, 354, 1280, 1, 0, 0, 0, 356, 1286, 1, 0, 0, 0, 358, 1292, 1, 0, 0, 0, 360, 361, 5, 100, 0, 0, 361, 362, 5, 105, 0, 0, 362, 363, 5, 115, 0, 0, 363, 364, 5, 115, 0, 0, 364, 365, 5, 101, 0, 0, 365, 366, 5, 99, 0, 0, 366, 367, 5, 116, 0, 0, 367, 368, 1, 0, 0, 0, 368, 369, 6, 0, 0, 0, 369, 15, 1, 0, 0, 0, 370, 371, 5, 100, 0, 0, 371, 372, 5, 114, 0, 0, 372, 373, 5, 111, 0, 0, 373, 374, 5, 112, 0, 0, 374, 375, 1, 0, 0, 0, 375, 376, 6, 1, 1, 0, 376, 17, 1, 0, 0, 0, 377, 378, 5, 101, 0, 0, 378, 379, 5, 110, 0, 0, 379, 380, 5, 114, 0, 0, 380, 381, 5, 105, 0, 0, 381, 382, 5, 99, 0, 0, 382, 383, 5, 104, 0, 0, 383, 384, 1, 0, 0, 0, 384, 385, 6, 2, 2, 0, 385, 19, 1, 0, 0, 0, 386, 387, 5, 101, 0, 0, 387, 388, 5, 118, 0, 0, 388, 389, 5, 97, 0, 0, 389, 390, 5, 108, 0, 0, 390, 391, 1, 0, 0, 0, 391, 392, 6, 3, 0, 0, 392, 21, 1, 0, 0, 0, 393, 394, 5, 101, 0, 0, 394, 395, 5, 120, 0, 0, 395, 396, 5, 112, 0, 0, 396, 397, 5, 108, 0, 0, 397, 398, 5, 97, 0, 0, 398, 399, 5, 105, 0, 0, 399, 400, 5, 110, 0, 0, 400, 401, 1, 0, 0, 0, 401, 402, 6, 4, 3, 0, 402, 23, 1, 0, 0, 0, 403, 404, 5, 102, 0, 0, 404, 405, 5, 114, 0, 0, 405, 406, 5, 111, 0, 0, 406, 407, 5, 109, 0, 0, 407, 408, 1, 0, 0, 0, 408, 409, 6, 5, 4, 0, 409, 25, 1, 0, 0, 0, 410, 411, 5, 103, 0, 0, 411, 412, 5, 114, 0, 0, 412, 413, 5, 111, 0, 0, 413, 414, 5, 107, 0, 0, 414, 415, 1, 0, 0, 0, 415, 416, 6, 6, 0, 0, 416, 27, 1, 0, 0, 0, 417, 418, 5, 105, 0, 0, 418, 419, 5, 110, 0, 0, 419, 420, 5, 108, 0, 0, 420, 421, 5, 105, 0, 0, 421, 422, 5, 110, 0, 0, 422, 423, 5, 101, 0, 0, 423, 424, 5, 115, 0, 0, 424, 425, 5, 116, 0, 0, 425, 426, 5, 97, 0, 0, 426, 427, 5, 116, 0, 0, 427, 428, 5, 115, 0, 0, 428, 429, 1, 0, 0, 0, 429, 430, 6, 7, 0, 0, 430, 29, 1, 0, 0, 0, 431, 432, 5, 107, 0, 0, 432, 433, 5, 101, 0, 0, 433, 434, 5, 101, 0, 0, 434, 435, 5, 112, 0, 0, 435, 436, 1, 0, 0, 0, 436, 437, 6, 8, 1, 0, 437, 31, 1, 0, 0, 0, 438, 439, 5, 108, 0, 0, 439, 440, 5, 105, 0, 0, 440, 441, 5, 109, 0, 0, 441, 442, 5, 105, 0, 0, 442, 443, 5, 116, 0, 0, 443, 444, 1, 0, 0, 0, 444, 445, 6, 9, 0, 0, 445, 33, 1, 0, 0, 0, 446, 447, 5, 109, 0, 0, 447, 448, 5, 101, 0, 0, 448, 449, 5, 116, 0, 0, 449, 450, 5, 97, 0, 0, 450, 451, 1, 0, 0, 0, 451, 452, 6, 10, 5, 0, 452, 35, 1, 0, 0, 0, 453, 454, 5, 109, 0, 0, 454, 455, 5, 101, 0, 0, 455, 456, 5, 116, 0, 0, 456, 457, 5, 114, 0, 0, 457, 458, 5, 105, 0, 0, 458, 459, 5, 99, 0, 0, 459, 460, 5, 115, 0, 0, 460, 461, 1, 0, 0, 0, 461, 462, 6, 11, 6, 0, 462, 37, 1, 0, 0, 0, 463, 464, 5, 109, 0, 0, 464, 465, 5, 118, 0, 0, 465, 466, 5, 95, 0, 0, 466, 467, 5, 101, 0, 0, 467, 468, 5, 120, 0, 0, 468, 469, 5, 112, 0, 0, 469, 470, 5, 97, 0, 0, 470, 471, 5, 110, 0, 0, 471, 472, 5, 100, 0, 0, 472, 473, 1, 0, 0, 0, 473, 474, 6, 12, 7, 0, 474, 39, 1, 0, 0, 0, 475, 476, 5, 114, 0, 0, 476, 477, 5, 101, 0, 0, 477, 478, 5, 110, 0, 0, 478, 479, 5, 97, 0, 0, 479, 480, 5, 109, 0, 0, 480, 481, 5, 101, 0, 0, 481, 482, 1, 0, 0, 0, 482, 483, 6, 13, 8, 0, 483, 41, 1, 0, 0, 0, 484, 485, 5, 114, 0, 0, 485, 486, 5, 111, 0, 0, 486, 487, 5, 119, 0, 0, 487, 488, 1, 0, 0, 0, 488, 489, 6, 14, 0, 0, 489, 43, 1, 0, 0, 0, 490, 491, 5, 115, 0, 0, 491, 492, 5, 104, 0, 0, 492, 493, 5, 111, 0, 0, 493, 494, 5, 119, 0, 0, 494, 495, 1, 0, 0, 0, 495, 496, 6, 15, 9, 0, 496, 45, 1, 0, 0, 0, 497, 498, 5, 115, 0, 0, 498, 499, 5, 111, 0, 0, 499, 500, 5, 114, 0, 0, 500, 501, 5, 116, 0, 0, 501, 502, 1, 0, 0, 0, 502, 503, 6, 16, 0, 0, 503, 47, 1, 0, 0, 0, 504, 505, 5, 115, 0, 0, 505, 506, 5, 116, 0, 0, 506, 507, 5, 97, 0, 0, 507, 508, 5, 116, 0, 0, 508, 509, 5, 115, 0, 0, 509, 510, 1, 0, 0, 0, 510, 511, 6, 17, 0, 0, 511, 49, 1, 0, 0, 0, 512, 513, 5, 119, 0, 0, 513, 514, 5, 104, 0, 0, 514, 515, 5, 101, 0, 0, 515, 516, 5, 114, 0, 0, 516, 517, 5, 101, 0, 0, 517, 518, 1, 0, 0, 0, 518, 519, 6, 18, 0, 0, 519, 51, 1, 0, 0, 0, 520, 522, 8, 0, 0, 0, 521, 520, 1, 0, 0, 0, 522, 523, 1, 0, 0, 0, 523, 521, 1, 0, 0, 0, 523, 524, 1, 0, 0, 0, 524, 525, 1, 0, 0, 0, 525, 526, 6, 19, 0, 0, 526, 53, 1, 0, 0, 0, 527, 528, 5, 47, 0, 0, 528, 529, 5, 47, 0, 0, 529, 533, 1, 0, 0, 0, 530, 532, 8, 1, 0, 0, 531, 530, 1, 0, 0, 0, 532, 535, 1, 0, 0, 0, 533, 531, 1, 0, 0, 0, 533, 534, 1, 0, 0, 0, 534, 537, 1, 0, 0, 0, 535, 533, 1, 0, 0, 0, 536, 538, 5, 13, 0, 0, 537, 536, 1, 0, 0, 0, 537, 538, 1, 0, 0, 0, 538, 540, 1, 0, 0, 0, 539, 541, 5, 10, 0, 0, 540, 539, 1, 0, 0, 0, 540, 541, 1, 0, 0, 0, 541, 542, 1, 0, 0, 0, 542, 543, 6, 20, 10, 0, 543, 55, 1, 0, 0, 0, 544, 545, 5, 47, 0, 0, 545, 546, 5, 42, 0, 0, 546, 551, 1, 0, 0, 0, 547, 550, 3, 56, 21, 0, 548, 550, 9, 0, 0, 0, 549, 547, 1, 0, 0, 0, 549, 548, 1, 0, 0, 0, 550, 553, 1, 0, 0, 0, 551, 552, 1, 0, 0, 0, 551, 549, 1, 0, 0, 0, 552, 554, 1, 0, 0, 0, 553, 551, 1, 0, 0, 0, 554, 555, 5, 42, 0, 0, 555, 556, 5, 47, 0, 0, 556, 557, 1, 0, 0, 0, 557, 558, 6, 21, 10, 0, 558, 57, 1, 0, 0, 0, 559, 561, 7, 2, 0, 0, 560, 559, 1, 0, 0, 0, 561, 562, 1, 0, 0, 0, 562, 560, 1, 0, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 565, 6, 22, 10, 0, 565, 59, 1, 0, 0, 0, 566, 570, 8, 3, 0, 0, 567, 568, 5, 47, 0, 0, 568, 570, 8, 4, 0, 0, 569, 566, 1, 0, 0, 0, 569, 567, 1, 0, 0, 0, 570, 61, 1, 0, 0, 0, 571, 573, 3, 60, 23, 0, 572, 571, 1, 0, 0, 0, 573, 574, 1, 0, 0, 0, 574, 572, 1, 0, 0, 0, 574, 575, 1, 0, 0, 0, 575, 63, 1, 0, 0, 0, 576, 577, 3, 172, 79, 0, 577, 578, 1, 0, 0, 0, 578, 579, 6, 25, 11, 0, 579, 580, 6, 25, 12, 0, 580, 65, 1, 0, 0, 0, 581, 582, 3, 74, 30, 0, 582, 583, 1, 0, 0, 0, 583, 584, 6, 26, 13, 0, 584, 585, 6, 26, 14, 0, 585, 67, 1, 0, 0, 0, 586, 587, 3, 58, 22, 0, 587, 588, 1, 0, 0, 0, 588, 589, 6, 27, 10, 0, 589, 69, 1, 0, 0, 0, 590, 591, 3, 54, 20, 0, 591, 592, 1, 0, 0, 0, 592, 593, 6, 28, 10, 0, 593, 71, 1, 0, 0, 0, 594, 595, 3, 56, 21, 0, 595, 596, 1, 0, 0, 0, 596, 597, 6, 29, 10, 0, 597, 73, 1, 0, 0, 0, 598, 599, 5, 124, 0, 0, 599, 600, 1, 0, 0, 0, 600, 601, 6, 30, 14, 0, 601, 75, 1, 0, 0, 0, 602, 603, 7, 5, 0, 0, 603, 77, 1, 0, 0, 0, 604, 605, 7, 6, 0, 0, 605, 79, 1, 0, 0, 0, 606, 607, 5, 92, 0, 0, 607, 608, 7, 7, 0, 0, 608, 81, 1, 0, 0, 0, 609, 610, 8, 8, 0, 0, 610, 83, 1, 0, 0, 0, 611, 613, 7, 9, 0, 0, 612, 614, 7, 10, 0, 0, 613, 612, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 616, 1, 0, 0, 0, 615, 617, 3, 76, 31, 0, 616, 615, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 616, 1, 0, 0, 0, 618, 619, 1, 0, 0, 0, 619, 85, 1, 0, 0, 0, 620, 621, 5, 64, 0, 0, 621, 87, 1, 0, 0, 0, 622, 623, 5, 96, 0, 0, 623, 89, 1, 0, 0, 0, 624, 628, 8, 11, 0, 0, 625, 626, 5, 96, 0, 0, 626, 628, 5, 96, 0, 0, 627, 624, 1, 0, 0, 0, 627, 625, 1, 0, 0, 0, 628, 91, 1, 0, 0, 0, 629, 630, 5, 95, 0, 0, 630, 93, 1, 0, 0, 0, 631, 635, 3, 78, 32, 0, 632, 635, 3, 76, 31, 0, 633, 635, 3, 92, 39, 0, 634, 631, 1, 0, 0, 0, 634, 632, 1, 0, 0, 0, 634, 633, 1, 0, 0, 0, 635, 95, 1, 0, 0, 0, 636, 641, 5, 34, 0, 0, 637, 640, 3, 80, 33, 0, 638, 640, 3, 82, 34, 0, 639, 637, 1, 0, 0, 0, 639, 638, 1, 0, 0, 0, 640, 643, 1, 0, 0, 0, 641, 639, 1, 0, 0, 0, 641, 642, 1, 0, 0, 0, 642, 644, 1, 0, 0, 0, 643, 641, 1, 0, 0, 0, 644, 666, 5, 34, 0, 0, 645, 646, 5, 34, 0, 0, 646, 647, 5, 34, 0, 0, 647, 648, 5, 34, 0, 0, 648, 652, 1, 0, 0, 0, 649, 651, 8, 1, 0, 0, 650, 649, 1, 0, 0, 0, 651, 654, 1, 0, 0, 0, 652, 653, 1, 0, 0, 0, 652, 650, 1, 0, 0, 0, 653, 655, 1, 0, 0, 0, 654, 652, 1, 0, 0, 0, 655, 656, 5, 34, 0, 0, 656, 657, 5, 34, 0, 0, 657, 658, 5, 34, 0, 0, 658, 660, 1, 0, 0, 0, 659, 661, 5, 34, 0, 0, 660, 659, 1, 0, 0, 0, 660, 661, 1, 0, 0, 0, 661, 663, 1, 0, 0, 0, 662, 664, 5, 34, 0, 0, 663, 662, 1, 0, 0, 0, 663, 664, 1, 0, 0, 0, 664, 666, 1, 0, 0, 0, 665, 636, 1, 0, 0, 0, 665, 645, 1, 0, 0, 0, 666, 97, 1, 0, 0, 0, 667, 669, 3, 76, 31, 0, 668, 667, 1, 0, 0, 0, 669, 670, 1, 0, 0, 0, 670, 668, 1, 0, 0, 0, 670, 671, 1, 0, 0, 0, 671, 99, 1, 0, 0, 0, 672, 674, 3, 76, 31, 0, 673, 672, 1, 0, 0, 0, 674, 675, 1, 0, 0, 0, 675, 673, 1, 0, 0, 0, 675, 676, 1, 0, 0, 0, 676, 677, 1, 0, 0, 0, 677, 681, 3, 116, 51, 0, 678, 680, 3, 76, 31, 0, 679, 678, 1, 0, 0, 0, 680, 683, 1, 0, 0, 0, 681, 679, 1, 0, 0, 0, 681, 682, 1, 0, 0, 0, 682, 715, 1, 0, 0, 0, 683, 681, 1, 0, 0, 0, 684, 686, 3, 116, 51, 0, 685, 687, 3, 76, 31, 0, 686, 685, 1, 0, 0, 0, 687, 688, 1, 0, 0, 0, 688, 686, 1, 0, 0, 0, 688, 689, 1, 0, 0, 0, 689, 715, 1, 0, 0, 0, 690, 692, 3, 76, 31, 0, 691, 690, 1, 0, 0, 0, 692, 693, 1, 0, 0, 0, 693, 691, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 702, 1, 0, 0, 0, 695, 699, 3, 116, 51, 0, 696, 698, 3, 76, 31, 0, 697, 696, 1, 0, 0, 0, 698, 701, 1, 0, 0, 0, 699, 697, 1, 0, 0, 0, 699, 700, 1, 0, 0, 0, 700, 703, 1, 0, 0, 0, 701, 699, 1, 0, 0, 0, 702, 695, 1, 0, 0, 0, 702, 703, 1, 0, 0, 0, 703, 704, 1, 0, 0, 0, 704, 705, 3, 84, 35, 0, 705, 715, 1, 0, 0, 0, 706, 708, 3, 116, 51, 0, 707, 709, 3, 76, 31, 0, 708, 707, 1, 0, 0, 0, 709, 710, 1, 0, 0, 0, 710, 708, 1, 0, 0, 0, 710, 711, 1, 0, 0, 0, 711, 712, 1, 0, 0, 0, 712, 713, 3, 84, 35, 0, 713, 715, 1, 0, 0, 0, 714, 673, 1, 0, 0, 0, 714, 684, 1, 0, 0, 0, 714, 691, 1, 0, 0, 0, 714, 706, 1, 0, 0, 0, 715, 101, 1, 0, 0, 0, 716, 717, 5, 98, 0, 0, 717, 718, 5, 121, 0, 0, 718, 103, 1, 0, 0, 0, 719, 720, 5, 97, 0, 0, 720, 721, 5, 110, 0, 0, 721, 722, 5, 100, 0, 0, 722, 105, 1, 0, 0, 0, 723, 724, 5, 97, 0, 0, 724, 725, 5, 115, 0, 0, 725, 726, 5, 99, 0, 0, 726, 107, 1, 0, 0, 0, 727, 728, 5, 61, 0, 0, 728, 109, 1, 0, 0, 0, 729, 730, 5, 58, 0, 0, 730, 731, 5, 58, 0, 0, 731, 111, 1, 0, 0, 0, 732, 733, 5, 44, 0, 0, 733, 113, 1, 0, 0, 0, 734, 735, 5, 100, 0, 0, 735, 736, 5, 101, 0, 0, 736, 737, 5, 115, 0, 0, 737, 738, 5, 99, 0, 0, 738, 115, 1, 0, 0, 0, 739, 740, 5, 46, 0, 0, 740, 117, 1, 0, 0, 0, 741, 742, 5, 102, 0, 0, 742, 743, 5, 97, 0, 0, 743, 744, 5, 108, 0, 0, 744, 745, 5, 115, 0, 0, 745, 746, 5, 101, 0, 0, 746, 119, 1, 0, 0, 0, 747, 748, 5, 102, 0, 0, 748, 749, 5, 105, 0, 0, 749, 750, 5, 114, 0, 0, 750, 751, 5, 115, 0, 0, 751, 752, 5, 116, 0, 0, 752, 121, 1, 0, 0, 0, 753, 754, 5, 108, 0, 0, 754, 755, 5, 97, 0, 0, 755, 756, 5, 115, 0, 0, 756, 757, 5, 116, 0, 0, 757, 123, 1, 0, 0, 0, 758, 759, 5, 40, 0, 0, 759, 125, 1, 0, 0, 0, 760, 761, 5, 105, 0, 0, 761, 762, 5, 110, 0, 0, 762, 127, 1, 0, 0, 0, 763, 764, 5, 105, 0, 0, 764, 765, 5, 115, 0, 0, 765, 129, 1, 0, 0, 0, 766, 767, 5, 108, 0, 0, 767, 768, 5, 105, 0, 0, 768, 769, 5, 107, 0, 0, 769, 770, 5, 101, 0, 0, 770, 131, 1, 0, 0, 0, 771, 772, 5, 110, 0, 0, 772, 773, 5, 111, 0, 0, 773, 774, 5, 116, 0, 0, 774, 133, 1, 0, 0, 0, 775, 776, 5, 110, 0, 0, 776, 777, 5, 117, 0, 0, 777, 778, 5, 108, 0, 0, 778, 779, 5, 108, 0, 0, 779, 135, 1, 0, 0, 0, 780, 781, 5, 110, 0, 0, 781, 782, 5, 117, 0, 0, 782, 783, 5, 108, 0, 0, 783, 784, 5, 108, 0, 0, 784, 785, 5, 115, 0, 0, 785, 137, 1, 0, 0, 0, 786, 787, 5, 111, 0, 0, 787, 788, 5, 114, 0, 0, 788, 139, 1, 0, 0, 0, 789, 790, 5, 63, 0, 0, 790, 141, 1, 0, 0, 0, 791, 792, 5, 114, 0, 0, 792, 793, 5, 108, 0, 0, 793, 794, 5, 105, 0, 0, 794, 795, 5, 107, 0, 0, 795, 796, 5, 101, 0, 0, 796, 143, 1, 0, 0, 0, 797, 798, 5, 41, 0, 0, 798, 145, 1, 0, 0, 0, 799, 800, 5, 116, 0, 0, 800, 801, 5, 114, 0, 0, 801, 802, 5, 117, 0, 0, 802, 803, 5, 101, 0, 0, 803, 147, 1, 0, 0, 0, 804, 805, 5, 61, 0, 0, 805, 806, 5, 61, 0, 0, 806, 149, 1, 0, 0, 0, 807, 808, 5, 61, 0, 0, 808, 809, 5, 126, 0, 0, 809, 151, 1, 0, 0, 0, 810, 811, 5, 33, 0, 0, 811, 812, 5, 61, 0, 0, 812, 153, 1, 0, 0, 0, 813, 814, 5, 60, 0, 0, 814, 155, 1, 0, 0, 0, 815, 816, 5, 60, 0, 0, 816, 817, 5, 61, 0, 0, 817, 157, 1, 0, 0, 0, 818, 819, 5, 62, 0, 0, 819, 159, 1, 0, 0, 0, 820, 821, 5, 62, 0, 0, 821, 822, 5, 61, 0, 0, 822, 161, 1, 0, 0, 0, 823, 824, 5, 43, 0, 0, 824, 163, 1, 0, 0, 0, 825, 826, 5, 45, 0, 0, 826, 165, 1, 0, 0, 0, 827, 828, 5, 42, 0, 0, 828, 167, 1, 0, 0, 0, 829, 830, 5, 47, 0, 0, 830, 169, 1, 0, 0, 0, 831, 832, 5, 37, 0, 0, 832, 171, 1, 0, 0, 0, 833, 834, 5, 91, 0, 0, 834, 835, 1, 0, 0, 0, 835, 836, 6, 79, 0, 0, 836, 837, 6, 79, 0, 0, 837, 173, 1, 0, 0, 0, 838, 839, 5, 93, 0, 0, 839, 840, 1, 0, 0, 0, 840, 841, 6, 80, 14, 0, 841, 842, 6, 80, 14, 0, 842, 175, 1, 0, 0, 0, 843, 847, 3, 78, 32, 0, 844, 846, 3, 94, 40, 0, 845, 844, 1, 0, 0, 0, 846, 849, 1, 0, 0, 0, 847, 845, 1, 0, 0, 0, 847, 848, 1, 0, 0, 0, 848, 860, 1, 0, 0, 0, 849, 847, 1, 0, 0, 0, 850, 853, 3, 92, 39, 0, 851, 853, 3, 86, 36, 0, 852, 850, 1, 0, 0, 0, 852, 851, 1, 0, 0, 0, 853, 855, 1, 0, 0, 0, 854, 856, 3, 94, 40, 0, 855, 854, 1, 0, 0, 0, 856, 857, 1, 0, 0, 0, 857, 855, 1, 0, 0, 0, 857, 858, 1, 0, 0, 0, 858, 860, 1, 0, 0, 0, 859, 843, 1, 0, 0, 0, 859, 852, 1, 0, 0, 0, 860, 177, 1, 0, 0, 0, 861, 863, 3, 88, 37, 0, 862, 864, 3, 90, 38, 0, 863, 862, 1, 0, 0, 0, 864, 865, 1, 0, 0, 0, 865, 863, 1, 0, 0, 0, 865, 866, 1, 0, 0, 0, 866, 867, 1, 0, 0, 0, 867, 868, 3, 88, 37, 0, 868, 179, 1, 0, 0, 0, 869, 870, 3, 178, 82, 0, 870, 181, 1, 0, 0, 0, 871, 872, 3, 54, 20, 0, 872, 873, 1, 0, 0, 0, 873, 874, 6, 84, 10, 0, 874, 183, 1, 0, 0, 0, 875, 876, 3, 56, 21, 0, 876, 877, 1, 0, 0, 0, 877, 878, 6, 85, 10, 0, 878, 185, 1, 0, 0, 0, 879, 880, 3, 58, 22, 0, 880, 881, 1, 0, 0, 0, 881, 882, 6, 86, 10, 0, 882, 187, 1, 0, 0, 0, 883, 884, 3, 74, 30, 0, 884, 885, 1, 0, 0, 0, 885, 886, 6, 87, 13, 0, 886, 887, 6, 87, 14, 0, 887, 189, 1, 0, 0, 0, 888, 889, 3, 172, 79, 0, 889, 890, 1, 0, 0, 0, 890, 891, 6, 88, 11, 0, 891, 191, 1, 0, 0, 0, 892, 893, 3, 174, 80, 0, 893, 894, 1, 0, 0, 0, 894, 895, 6, 89, 15, 0, 895, 193, 1, 0, 0, 0, 896, 897, 3, 112, 49, 0, 897, 898, 1, 0, 0, 0, 898, 899, 6, 90, 16, 0, 899, 195, 1, 0, 0, 0, 900, 901, 3, 108, 47, 0, 901, 902, 1, 0, 0, 0, 902, 903, 6, 91, 17, 0, 903, 197, 1, 0, 0, 0, 904, 905, 3, 96, 41, 0, 905, 906, 1, 0, 0, 0, 906, 907, 6, 92, 18, 0, 907, 199, 1, 0, 0, 0, 908, 909, 5, 109, 0, 0, 909, 910, 5, 101, 0, 0, 910, 911, 5, 116, 0, 0, 911, 912, 5, 97, 0, 0, 912, 913, 5, 100, 0, 0, 913, 914, 5, 97, 0, 0, 914, 915, 5, 116, 0, 0, 915, 916, 5, 97, 0, 0, 916, 201, 1, 0, 0, 0, 917, 918, 3, 62, 24, 0, 918, 919, 1, 0, 0, 0, 919, 920, 6, 94, 19, 0, 920, 203, 1, 0, 0, 0, 921, 922, 3, 54, 20, 0, 922, 923, 1, 0, 0, 0, 923, 924, 6, 95, 10, 0, 924, 205, 1, 0, 0, 0, 925, 926, 3, 56, 21, 0, 926, 927, 1, 0, 0, 0, 927, 928, 6, 96, 10, 0, 928, 207, 1, 0, 0, 0, 929, 930, 3, 58, 22, 0, 930, 931, 1, 0, 0, 0, 931, 932, 6, 97, 10, 0, 932, 209, 1, 0, 0, 0, 933, 934, 3, 74, 30, 0, 934, 935, 1, 0, 0, 0, 935, 936, 6, 98, 13, 0, 936, 937, 6, 98, 14, 0, 937, 211, 1, 0, 0, 0, 938, 939, 3, 116, 51, 0, 939, 940, 1, 0, 0, 0, 940, 941, 6, 99, 20, 0, 941, 213, 1, 0, 0, 0, 942, 943, 3, 112, 49, 0, 943, 944, 1, 0, 0, 0, 944, 945, 6, 100, 16, 0, 945, 215, 1, 0, 0, 0, 946, 951, 3, 78, 32, 0, 947, 951, 3, 76, 31, 0, 948, 951, 3, 92, 39, 0, 949, 951, 3, 166, 76, 0, 950, 946, 1, 0, 0, 0, 950, 947, 1, 0, 0, 0, 950, 948, 1, 0, 0, 0, 950, 949, 1, 0, 0, 0, 951, 217, 1, 0, 0, 0, 952, 955, 3, 78, 32, 0, 953, 955, 3, 166, 76, 0, 954, 952, 1, 0, 0, 0, 954, 953, 1, 0, 0, 0, 955, 959, 1, 0, 0, 0, 956, 958, 3, 216, 101, 0, 957, 956, 1, 0, 0, 0, 958, 961, 1, 0, 0, 0, 959, 957, 1, 0, 0, 0, 959, 960, 1, 0, 0, 0, 960, 972, 1, 0, 0, 0, 961, 959, 1, 0, 0, 0, 962, 965, 3, 92, 39, 0, 963, 965, 3, 86, 36, 0, 964, 962, 1, 0, 0, 0, 964, 963, 1, 0, 0, 0, 965, 967, 1, 0, 0, 0, 966, 968, 3, 216, 101, 0, 967, 966, 1, 0, 0, 0, 968, 969, 1, 0, 0, 0, 969, 967, 1, 0, 0, 0, 969, 970, 1, 0, 0, 0, 970, 972, 1, 0, 0, 0, 971, 954, 1, 0, 0, 0, 971, 964, 1, 0, 0, 0, 972, 219, 1, 0, 0, 0, 973, 976, 3, 218, 102, 0, 974, 976, 3, 178, 82, 0, 975, 973, 1, 0, 0, 0, 975, 974, 1, 0, 0, 0, 976, 977, 1, 0, 0, 0, 977, 975, 1, 0, 0, 0, 977, 978, 1, 0, 0, 0, 978, 221, 1, 0, 0, 0, 979, 980, 3, 54, 20, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 104, 10, 0, 982, 223, 1, 0, 0, 0, 983, 984, 3, 56, 21, 0, 984, 985, 1, 0, 0, 0, 985, 986, 6, 105, 10, 0, 986, 225, 1, 0, 0, 0, 987, 988, 3, 58, 22, 0, 988, 989, 1, 0, 0, 0, 989, 990, 6, 106, 10, 0, 990, 227, 1, 0, 0, 0, 991, 992, 3, 74, 30, 0, 992, 993, 1, 0, 0, 0, 993, 994, 6, 107, 13, 0, 994, 995, 6, 107, 14, 0, 995, 229, 1, 0, 0, 0, 996, 997, 3, 108, 47, 0, 997, 998, 1, 0, 0, 0, 998, 999, 6, 108, 17, 0, 999, 231, 1, 0, 0, 0, 1000, 1001, 3, 112, 49, 0, 1001, 1002, 1, 0, 0, 0, 1002, 1003, 6, 109, 16, 0, 1003, 233, 1, 0, 0, 0, 1004, 1005, 3, 116, 51, 0, 1005, 1006, 1, 0, 0, 0, 1006, 1007, 6, 110, 20, 0, 1007, 235, 1, 0, 0, 0, 1008, 1009, 5, 97, 0, 0, 1009, 1010, 5, 115, 0, 0, 1010, 237, 1, 0, 0, 0, 1011, 1012, 3, 220, 103, 0, 1012, 1013, 1, 0, 0, 0, 1013, 1014, 6, 112, 21, 0, 1014, 239, 1, 0, 0, 0, 1015, 1016, 3, 54, 20, 0, 1016, 1017, 1, 0, 0, 0, 1017, 1018, 6, 113, 10, 0, 1018, 241, 1, 0, 0, 0, 1019, 1020, 3, 56, 21, 0, 1020, 1021, 1, 0, 0, 0, 1021, 1022, 6, 114, 10, 0, 1022, 243, 1, 0, 0, 0, 1023, 1024, 3, 58, 22, 0, 1024, 1025, 1, 0, 0, 0, 1025, 1026, 6, 115, 10, 0, 1026, 245, 1, 0, 0, 0, 1027, 1028, 3, 74, 30, 0, 1028, 1029, 1, 0, 0, 0, 1029, 1030, 6, 116, 13, 0, 1030, 1031, 6, 116, 14, 0, 1031, 247, 1, 0, 0, 0, 1032, 1033, 3, 172, 79, 0, 1033, 1034, 1, 0, 0, 0, 1034, 1035, 6, 117, 11, 0, 1035, 1036, 6, 117, 22, 0, 1036, 249, 1, 0, 0, 0, 1037, 1038, 5, 111, 0, 0, 1038, 1039, 5, 110, 0, 0, 1039, 1040, 1, 0, 0, 0, 1040, 1041, 6, 118, 23, 0, 1041, 251, 1, 0, 0, 0, 1042, 1043, 5, 119, 0, 0, 1043, 1044, 5, 105, 0, 0, 1044, 1045, 5, 116, 0, 0, 1045, 1046, 5, 104, 0, 0, 1046, 1047, 1, 0, 0, 0, 1047, 1048, 6, 119, 23, 0, 1048, 253, 1, 0, 0, 0, 1049, 1050, 8, 12, 0, 0, 1050, 255, 1, 0, 0, 0, 1051, 1053, 3, 254, 120, 0, 1052, 1051, 1, 0, 0, 0, 1053, 1054, 1, 0, 0, 0, 1054, 1052, 1, 0, 0, 0, 1054, 1055, 1, 0, 0, 0, 1055, 1056, 1, 0, 0, 0, 1056, 1057, 3, 324, 155, 0, 1057, 1059, 1, 0, 0, 0, 1058, 1052, 1, 0, 0, 0, 1058, 1059, 1, 0, 0, 0, 1059, 1061, 1, 0, 0, 0, 1060, 1062, 3, 254, 120, 0, 1061, 1060, 1, 0, 0, 0, 1062, 1063, 1, 0, 0, 0, 1063, 1061, 1, 0, 0, 0, 1063, 1064, 1, 0, 0, 0, 1064, 257, 1, 0, 0, 0, 1065, 1066, 3, 180, 83, 0, 1066, 1067, 1, 0, 0, 0, 1067, 1068, 6, 122, 24, 0, 1068, 259, 1, 0, 0, 0, 1069, 1070, 3, 256, 121, 0, 1070, 1071, 1, 0, 0, 0, 1071, 1072, 6, 123, 25, 0, 1072, 261, 1, 0, 0, 0, 1073, 1074, 3, 54, 20, 0, 1074, 1075, 1, 0, 0, 0, 1075, 1076, 6, 124, 10, 0, 1076, 263, 1, 0, 0, 0, 1077, 1078, 3, 56, 21, 0, 1078, 1079, 1, 0, 0, 0, 1079, 1080, 6, 125, 10, 0, 1080, 265, 1, 0, 0, 0, 1081, 1082, 3, 58, 22, 0, 1082, 1083, 1, 0, 0, 0, 1083, 1084, 6, 126, 10, 0, 1084, 267, 1, 0, 0, 0, 1085, 1086, 3, 74, 30, 0, 1086, 1087, 1, 0, 0, 0, 1087, 1088, 6, 127, 13, 0, 1088, 1089, 6, 127, 14, 0, 1089, 1090, 6, 127, 14, 0, 1090, 269, 1, 0, 0, 0, 1091, 1092, 3, 108, 47, 0, 1092, 1093, 1, 0, 0, 0, 1093, 1094, 6, 128, 17, 0, 1094, 271, 1, 0, 0, 0, 1095, 1096, 3, 112, 49, 0, 1096, 1097, 1, 0, 0, 0, 1097, 1098, 6, 129, 16, 0, 1098, 273, 1, 0, 0, 0, 1099, 1100, 3, 116, 51, 0, 1100, 1101, 1, 0, 0, 0, 1101, 1102, 6, 130, 20, 0, 1102, 275, 1, 0, 0, 0, 1103, 1104, 3, 252, 119, 0, 1104, 1105, 1, 0, 0, 0, 1105, 1106, 6, 131, 26, 0, 1106, 277, 1, 0, 0, 0, 1107, 1108, 3, 220, 103, 0, 1108, 1109, 1, 0, 0, 0, 1109, 1110, 6, 132, 21, 0, 1110, 279, 1, 0, 0, 0, 1111, 1112, 3, 180, 83, 0, 1112, 1113, 1, 0, 0, 0, 1113, 1114, 6, 133, 24, 0, 1114, 281, 1, 0, 0, 0, 1115, 1116, 3, 54, 20, 0, 1116, 1117, 1, 0, 0, 0, 1117, 1118, 6, 134, 10, 0, 1118, 283, 1, 0, 0, 0, 1119, 1120, 3, 56, 21, 0, 1120, 1121, 1, 0, 0, 0, 1121, 1122, 6, 135, 10, 0, 1122, 285, 1, 0, 0, 0, 1123, 1124, 3, 58, 22, 0, 1124, 1125, 1, 0, 0, 0, 1125, 1126, 6, 136, 10, 0, 1126, 287, 1, 0, 0, 0, 1127, 1128, 3, 74, 30, 0, 1128, 1129, 1, 0, 0, 0, 1129, 1130, 6, 137, 13, 0, 1130, 1131, 6, 137, 14, 0, 1131, 289, 1, 0, 0, 0, 1132, 1133, 3, 116, 51, 0, 1133, 1134, 1, 0, 0, 0, 1134, 1135, 6, 138, 20, 0, 1135, 291, 1, 0, 0, 0, 1136, 1137, 3, 180, 83, 0, 1137, 1138, 1, 0, 0, 0, 1138, 1139, 6, 139, 24, 0, 1139, 293, 1, 0, 0, 0, 1140, 1141, 3, 176, 81, 0, 1141, 1142, 1, 0, 0, 0, 1142, 1143, 6, 140, 27, 0, 1143, 295, 1, 0, 0, 0, 1144, 1145, 3, 54, 20, 0, 1145, 1146, 1, 0, 0, 0, 1146, 1147, 6, 141, 10, 0, 1147, 297, 1, 0, 0, 0, 1148, 1149, 3, 56, 21, 0, 1149, 1150, 1, 0, 0, 0, 1150, 1151, 6, 142, 10, 0, 1151, 299, 1, 0, 0, 0, 1152, 1153, 3, 58, 22, 0, 1153, 1154, 1, 0, 0, 0, 1154, 1155, 6, 143, 10, 0, 1155, 301, 1, 0, 0, 0, 1156, 1157, 3, 74, 30, 0, 1157, 1158, 1, 0, 0, 0, 1158, 1159, 6, 144, 13, 0, 1159, 1160, 6, 144, 14, 0, 1160, 303, 1, 0, 0, 0, 1161, 1162, 5, 105, 0, 0, 1162, 1163, 5, 110, 0, 0, 1163, 1164, 5, 102, 0, 0, 1164, 1165, 5, 111, 0, 0, 1165, 305, 1, 0, 0, 0, 1166, 1167, 3, 54, 20, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1169, 6, 146, 10, 0, 1169, 307, 1, 0, 0, 0, 1170, 1171, 3, 56, 21, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1173, 6, 147, 10, 0, 1173, 309, 1, 0, 0, 0, 1174, 1175, 3, 58, 22, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1177, 6, 148, 10, 0, 1177, 311, 1, 0, 0, 0, 1178, 1179, 3, 74, 30, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 149, 13, 0, 1181, 1182, 6, 149, 14, 0, 1182, 313, 1, 0, 0, 0, 1183, 1184, 5, 102, 0, 0, 1184, 1185, 5, 117, 0, 0, 1185, 1186, 5, 110, 0, 0, 1186, 1187, 5, 99, 0, 0, 1187, 1188, 5, 116, 0, 0, 1188, 1189, 5, 105, 0, 0, 1189, 1190, 5, 111, 0, 0, 1190, 1191, 5, 110, 0, 0, 1191, 1192, 5, 115, 0, 0, 1192, 315, 1, 0, 0, 0, 1193, 1194, 3, 54, 20, 0, 1194, 1195, 1, 0, 0, 0, 1195, 1196, 6, 151, 10, 0, 1196, 317, 1, 0, 0, 0, 1197, 1198, 3, 56, 21, 0, 1198, 1199, 1, 0, 0, 0, 1199, 1200, 6, 152, 10, 0, 1200, 319, 1, 0, 0, 0, 1201, 1202, 3, 58, 22, 0, 1202, 1203, 1, 0, 0, 0, 1203, 1204, 6, 153, 10, 0, 1204, 321, 1, 0, 0, 0, 1205, 1206, 3, 174, 80, 0, 1206, 1207, 1, 0, 0, 0, 1207, 1208, 6, 154, 15, 0, 1208, 1209, 6, 154, 14, 0, 1209, 323, 1, 0, 0, 0, 1210, 1211, 5, 58, 0, 0, 1211, 325, 1, 0, 0, 0, 1212, 1218, 3, 86, 36, 0, 1213, 1218, 3, 76, 31, 0, 1214, 1218, 3, 116, 51, 0, 1215, 1218, 3, 78, 32, 0, 1216, 1218, 3, 92, 39, 0, 1217, 1212, 1, 0, 0, 0, 1217, 1213, 1, 0, 0, 0, 1217, 1214, 1, 0, 0, 0, 1217, 1215, 1, 0, 0, 0, 1217, 1216, 1, 0, 0, 0, 1218, 1219, 1, 0, 0, 0, 1219, 1217, 1, 0, 0, 0, 1219, 1220, 1, 0, 0, 0, 1220, 327, 1, 0, 0, 0, 1221, 1222, 3, 54, 20, 0, 1222, 1223, 1, 0, 0, 0, 1223, 1224, 6, 157, 10, 0, 1224, 329, 1, 0, 0, 0, 1225, 1226, 3, 56, 21, 0, 1226, 1227, 1, 0, 0, 0, 1227, 1228, 6, 158, 10, 0, 1228, 331, 1, 0, 0, 0, 1229, 1230, 3, 58, 22, 0, 1230, 1231, 1, 0, 0, 0, 1231, 1232, 6, 159, 10, 0, 1232, 333, 1, 0, 0, 0, 1233, 1234, 3, 74, 30, 0, 1234, 1235, 1, 0, 0, 0, 1235, 1236, 6, 160, 13, 0, 1236, 1237, 6, 160, 14, 0, 1237, 335, 1, 0, 0, 0, 1238, 1239, 3, 62, 24, 0, 1239, 1240, 1, 0, 0, 0, 1240, 1241, 6, 161, 19, 0, 1241, 1242, 6, 161, 14, 0, 1242, 1243, 6, 161, 28, 0, 1243, 337, 1, 0, 0, 0, 1244, 1245, 3, 54, 20, 0, 1245, 1246, 1, 0, 0, 0, 1246, 1247, 6, 162, 10, 0, 1247, 339, 1, 0, 0, 0, 1248, 1249, 3, 56, 21, 0, 1249, 1250, 1, 0, 0, 0, 1250, 1251, 6, 163, 10, 0, 1251, 341, 1, 0, 0, 0, 1252, 1253, 3, 58, 22, 0, 1253, 1254, 1, 0, 0, 0, 1254, 1255, 6, 164, 10, 0, 1255, 343, 1, 0, 0, 0, 1256, 1257, 3, 112, 49, 0, 1257, 1258, 1, 0, 0, 0, 1258, 1259, 6, 165, 16, 0, 1259, 1260, 6, 165, 14, 0, 1260, 1261, 6, 165, 6, 0, 1261, 345, 1, 0, 0, 0, 1262, 1263, 3, 54, 20, 0, 1263, 1264, 1, 0, 0, 0, 1264, 1265, 6, 166, 10, 0, 1265, 347, 1, 0, 0, 0, 1266, 1267, 3, 56, 21, 0, 1267, 1268, 1, 0, 0, 0, 1268, 1269, 6, 167, 10, 0, 1269, 349, 1, 0, 0, 0, 1270, 1271, 3, 58, 22, 0, 1271, 1272, 1, 0, 0, 0, 1272, 1273, 6, 168, 10, 0, 1273, 351, 1, 0, 0, 0, 1274, 1275, 3, 180, 83, 0, 1275, 1276, 1, 0, 0, 0, 1276, 1277, 6, 169, 14, 0, 1277, 1278, 6, 169, 0, 0, 1278, 1279, 6, 169, 24, 0, 1279, 353, 1, 0, 0, 0, 1280, 1281, 3, 176, 81, 0, 1281, 1282, 1, 0, 0, 0, 1282, 1283, 6, 170, 14, 0, 1283, 1284, 6, 170, 0, 0, 1284, 1285, 6, 170, 27, 0, 1285, 355, 1, 0, 0, 0, 1286, 1287, 3, 102, 44, 0, 1287, 1288, 1, 0, 0, 0, 1288, 1289, 6, 171, 14, 0, 1289, 1290, 6, 171, 0, 0, 1290, 1291, 6, 171, 29, 0, 1291, 357, 1, 0, 0, 0, 1292, 1293, 3, 74, 30, 0, 1293, 1294, 1, 0, 0, 0, 1294, 1295, 6, 172, 13, 0, 1295, 1296, 6, 172, 14, 0, 1296, 359, 1, 0, 0, 0, 60, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 523, 533, 537, 540, 549, 551, 562, 569, 574, 613, 618, 627, 634, 639, 641, 652, 660, 663, 665, 670, 675, 681, 688, 693, 699, 702, 710, 714, 847, 852, 857, 859, 865, 950, 954, 959, 964, 969, 971, 975, 977, 1054, 1058, 1063, 1217, 1219, 30, 5, 2, 0, 5, 4, 0, 5, 6, 0, 5, 1, 0, 5, 3, 0, 5, 10, 0, 5, 12, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 0, 1, 0, 7, 67, 0, 5, 0, 0, 7, 28, 0, 4, 0, 0, 7, 68, 0, 7, 37, 0, 7, 35, 0, 7, 29, 0, 7, 24, 0, 7, 39, 0, 7, 78, 0, 5, 11, 0, 5, 7, 0, 7, 70, 0, 7, 88, 0, 7, 87, 0, 7, 69, 0, 5, 13, 0, 7, 32, 0]
\ No newline at end of file
+[4, 0, 123, 1404, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 4, 20, 565, 8, 20, 11, 20, 12, 20, 566, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 575, 8, 21, 10, 21, 12, 21, 578, 9, 21, 1, 21, 3, 21, 581, 8, 21, 1, 21, 3, 21, 584, 8, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 593, 8, 22, 10, 22, 12, 22, 596, 9, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 4, 23, 604, 8, 23, 11, 23, 12, 23, 605, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 3, 24, 613, 8, 24, 1, 25, 4, 25, 616, 8, 25, 11, 25, 12, 25, 617, 1, 26, 1, 26, 1, 26, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 28, 1, 29, 1, 29, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 1, 35, 1, 35, 1, 36, 1, 36, 3, 36, 657, 8, 36, 1, 36, 4, 36, 660, 8, 36, 11, 36, 12, 36, 661, 1, 37, 1, 37, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 3, 39, 671, 8, 39, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 3, 41, 678, 8, 41, 1, 42, 1, 42, 1, 42, 5, 42, 683, 8, 42, 10, 42, 12, 42, 686, 9, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 694, 8, 42, 10, 42, 12, 42, 697, 9, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 3, 42, 704, 8, 42, 1, 42, 3, 42, 707, 8, 42, 3, 42, 709, 8, 42, 1, 43, 4, 43, 712, 8, 43, 11, 43, 12, 43, 713, 1, 44, 4, 44, 717, 8, 44, 11, 44, 12, 44, 718, 1, 44, 1, 44, 5, 44, 723, 8, 44, 10, 44, 12, 44, 726, 9, 44, 1, 44, 1, 44, 4, 44, 730, 8, 44, 11, 44, 12, 44, 731, 1, 44, 4, 44, 735, 8, 44, 11, 44, 12, 44, 736, 1, 44, 1, 44, 5, 44, 741, 8, 44, 10, 44, 12, 44, 744, 9, 44, 3, 44, 746, 8, 44, 1, 44, 1, 44, 1, 44, 1, 44, 4, 44, 752, 8, 44, 11, 44, 12, 44, 753, 1, 44, 1, 44, 3, 44, 758, 8, 44, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 69, 1, 70, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 76, 1, 76, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 5, 82, 889, 8, 82, 10, 82, 12, 82, 892, 9, 82, 1, 82, 1, 82, 3, 82, 896, 8, 82, 1, 82, 4, 82, 899, 8, 82, 11, 82, 12, 82, 900, 3, 82, 903, 8, 82, 1, 83, 1, 83, 4, 83, 907, 8, 83, 11, 83, 12, 83, 908, 1, 83, 1, 83, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 3, 102, 994, 8, 102, 1, 103, 1, 103, 3, 103, 998, 8, 103, 1, 103, 5, 103, 1001, 8, 103, 10, 103, 12, 103, 1004, 9, 103, 1, 103, 1, 103, 3, 103, 1008, 8, 103, 1, 103, 4, 103, 1011, 8, 103, 11, 103, 12, 103, 1012, 3, 103, 1015, 8, 103, 1, 104, 1, 104, 4, 104, 1019, 8, 104, 11, 104, 12, 104, 1020, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 122, 4, 122, 1096, 8, 122, 11, 122, 12, 122, 1097, 1, 122, 1, 122, 3, 122, 1102, 8, 122, 1, 122, 4, 122, 1105, 8, 122, 11, 122, 12, 122, 1106, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 172, 4, 172, 1325, 8, 172, 11, 172, 12, 172, 1326, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 188, 2, 594, 695, 0, 189, 16, 1, 18, 2, 20, 3, 22, 4, 24, 5, 26, 6, 28, 7, 30, 8, 32, 9, 34, 10, 36, 11, 38, 12, 40, 13, 42, 14, 44, 15, 46, 16, 48, 17, 50, 18, 52, 19, 54, 20, 56, 21, 58, 22, 60, 23, 62, 24, 64, 0, 66, 25, 68, 0, 70, 0, 72, 26, 74, 27, 76, 28, 78, 29, 80, 0, 82, 0, 84, 0, 86, 0, 88, 0, 90, 0, 92, 0, 94, 0, 96, 0, 98, 0, 100, 30, 102, 31, 104, 32, 106, 33, 108, 34, 110, 35, 112, 36, 114, 37, 116, 38, 118, 39, 120, 40, 122, 41, 124, 42, 126, 43, 128, 44, 130, 45, 132, 46, 134, 47, 136, 48, 138, 49, 140, 50, 142, 51, 144, 52, 146, 53, 148, 54, 150, 55, 152, 56, 154, 57, 156, 58, 158, 59, 160, 60, 162, 61, 164, 62, 166, 63, 168, 64, 170, 65, 172, 66, 174, 67, 176, 68, 178, 69, 180, 70, 182, 0, 184, 71, 186, 72, 188, 73, 190, 74, 192, 0, 194, 0, 196, 0, 198, 0, 200, 0, 202, 0, 204, 75, 206, 0, 208, 76, 210, 77, 212, 78, 214, 0, 216, 0, 218, 0, 220, 0, 222, 0, 224, 79, 226, 80, 228, 81, 230, 82, 232, 0, 234, 0, 236, 0, 238, 0, 240, 83, 242, 0, 244, 84, 246, 85, 248, 86, 250, 0, 252, 0, 254, 87, 256, 88, 258, 0, 260, 89, 262, 0, 264, 0, 266, 90, 268, 91, 270, 92, 272, 0, 274, 0, 276, 0, 278, 0, 280, 0, 282, 0, 284, 0, 286, 93, 288, 94, 290, 95, 292, 0, 294, 0, 296, 0, 298, 0, 300, 0, 302, 96, 304, 97, 306, 98, 308, 0, 310, 0, 312, 0, 314, 0, 316, 99, 318, 100, 320, 101, 322, 0, 324, 0, 326, 0, 328, 0, 330, 102, 332, 103, 334, 104, 336, 0, 338, 105, 340, 106, 342, 107, 344, 108, 346, 0, 348, 109, 350, 110, 352, 111, 354, 112, 356, 0, 358, 113, 360, 114, 362, 115, 364, 116, 366, 117, 368, 0, 370, 0, 372, 118, 374, 119, 376, 120, 378, 0, 380, 121, 382, 122, 384, 123, 386, 0, 388, 0, 390, 0, 392, 0, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 13, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 10, 0, 9, 10, 13, 13, 32, 32, 44, 44, 47, 47, 61, 61, 91, 91, 93, 93, 96, 96, 124, 124, 2, 0, 42, 42, 47, 47, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 5, 0, 34, 34, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 69, 69, 101, 101, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1427, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 52, 1, 0, 0, 0, 0, 54, 1, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 58, 1, 0, 0, 0, 0, 60, 1, 0, 0, 0, 0, 62, 1, 0, 0, 0, 0, 66, 1, 0, 0, 0, 1, 68, 1, 0, 0, 0, 1, 70, 1, 0, 0, 0, 1, 72, 1, 0, 0, 0, 1, 74, 1, 0, 0, 0, 1, 76, 1, 0, 0, 0, 2, 78, 1, 0, 0, 0, 2, 100, 1, 0, 0, 0, 2, 102, 1, 0, 0, 0, 2, 104, 1, 0, 0, 0, 2, 106, 1, 0, 0, 0, 2, 108, 1, 0, 0, 0, 2, 110, 1, 0, 0, 0, 2, 112, 1, 0, 0, 0, 2, 114, 1, 0, 0, 0, 2, 116, 1, 0, 0, 0, 2, 118, 1, 0, 0, 0, 2, 120, 1, 0, 0, 0, 2, 122, 1, 0, 0, 0, 2, 124, 1, 0, 0, 0, 2, 126, 1, 0, 0, 0, 2, 128, 1, 0, 0, 0, 2, 130, 1, 0, 0, 0, 2, 132, 1, 0, 0, 0, 2, 134, 1, 0, 0, 0, 2, 136, 1, 0, 0, 0, 2, 138, 1, 0, 0, 0, 2, 140, 1, 0, 0, 0, 2, 142, 1, 0, 0, 0, 2, 144, 1, 0, 0, 0, 2, 146, 1, 0, 0, 0, 2, 148, 1, 0, 0, 0, 2, 150, 1, 0, 0, 0, 2, 152, 1, 0, 0, 0, 2, 154, 1, 0, 0, 0, 2, 156, 1, 0, 0, 0, 2, 158, 1, 0, 0, 0, 2, 160, 1, 0, 0, 0, 2, 162, 1, 0, 0, 0, 2, 164, 1, 0, 0, 0, 2, 166, 1, 0, 0, 0, 2, 168, 1, 0, 0, 0, 2, 170, 1, 0, 0, 0, 2, 172, 1, 0, 0, 0, 2, 174, 1, 0, 0, 0, 2, 176, 1, 0, 0, 0, 2, 178, 1, 0, 0, 0, 2, 180, 1, 0, 0, 0, 2, 184, 1, 0, 0, 0, 2, 186, 1, 0, 0, 0, 2, 188, 1, 0, 0, 0, 2, 190, 1, 0, 0, 0, 3, 192, 1, 0, 0, 0, 3, 194, 1, 0, 0, 0, 3, 196, 1, 0, 0, 0, 3, 198, 1, 0, 0, 0, 3, 200, 1, 0, 0, 0, 3, 202, 1, 0, 0, 0, 3, 204, 1, 0, 0, 0, 3, 206, 1, 0, 0, 0, 3, 208, 1, 0, 0, 0, 3, 210, 1, 0, 0, 0, 3, 212, 1, 0, 0, 0, 4, 214, 1, 0, 0, 0, 4, 216, 1, 0, 0, 0, 4, 218, 1, 0, 0, 0, 4, 224, 1, 0, 0, 0, 4, 226, 1, 0, 0, 0, 4, 228, 1, 0, 0, 0, 4, 230, 1, 0, 0, 0, 5, 232, 1, 0, 0, 0, 5, 234, 1, 0, 0, 0, 5, 236, 1, 0, 0, 0, 5, 238, 1, 0, 0, 0, 5, 240, 1, 0, 0, 0, 5, 242, 1, 0, 0, 0, 5, 244, 1, 0, 0, 0, 5, 246, 1, 0, 0, 0, 5, 248, 1, 0, 0, 0, 6, 250, 1, 0, 0, 0, 6, 252, 1, 0, 0, 0, 6, 254, 1, 0, 0, 0, 6, 256, 1, 0, 0, 0, 6, 260, 1, 0, 0, 0, 6, 262, 1, 0, 0, 0, 6, 264, 1, 0, 0, 0, 6, 266, 1, 0, 0, 0, 6, 268, 1, 0, 0, 0, 6, 270, 1, 0, 0, 0, 7, 272, 1, 0, 0, 0, 7, 274, 1, 0, 0, 0, 7, 276, 1, 0, 0, 0, 7, 278, 1, 0, 0, 0, 7, 280, 1, 0, 0, 0, 7, 282, 1, 0, 0, 0, 7, 284, 1, 0, 0, 0, 7, 286, 1, 0, 0, 0, 7, 288, 1, 0, 0, 0, 7, 290, 1, 0, 0, 0, 8, 292, 1, 0, 0, 0, 8, 294, 1, 0, 0, 0, 8, 296, 1, 0, 0, 0, 8, 298, 1, 0, 0, 0, 8, 300, 1, 0, 0, 0, 8, 302, 1, 0, 0, 0, 8, 304, 1, 0, 0, 0, 8, 306, 1, 0, 0, 0, 9, 308, 1, 0, 0, 0, 9, 310, 1, 0, 0, 0, 9, 312, 1, 0, 0, 0, 9, 314, 1, 0, 0, 0, 9, 316, 1, 0, 0, 0, 9, 318, 1, 0, 0, 0, 9, 320, 1, 0, 0, 0, 10, 322, 1, 0, 0, 0, 10, 324, 1, 0, 0, 0, 10, 326, 1, 0, 0, 0, 10, 328, 1, 0, 0, 0, 10, 330, 1, 0, 0, 0, 10, 332, 1, 0, 0, 0, 10, 334, 1, 0, 0, 0, 11, 336, 1, 0, 0, 0, 11, 338, 1, 0, 0, 0, 11, 340, 1, 0, 0, 0, 11, 342, 1, 0, 0, 0, 11, 344, 1, 0, 0, 0, 12, 346, 1, 0, 0, 0, 12, 348, 1, 0, 0, 0, 12, 350, 1, 0, 0, 0, 12, 352, 1, 0, 0, 0, 12, 354, 1, 0, 0, 0, 13, 356, 1, 0, 0, 0, 13, 358, 1, 0, 0, 0, 13, 360, 1, 0, 0, 0, 13, 362, 1, 0, 0, 0, 13, 364, 1, 0, 0, 0, 13, 366, 1, 0, 0, 0, 14, 368, 1, 0, 0, 0, 14, 370, 1, 0, 0, 0, 14, 372, 1, 0, 0, 0, 14, 374, 1, 0, 0, 0, 14, 376, 1, 0, 0, 0, 15, 378, 1, 0, 0, 0, 15, 380, 1, 0, 0, 0, 15, 382, 1, 0, 0, 0, 15, 384, 1, 0, 0, 0, 15, 386, 1, 0, 0, 0, 15, 388, 1, 0, 0, 0, 15, 390, 1, 0, 0, 0, 15, 392, 1, 0, 0, 0, 16, 394, 1, 0, 0, 0, 18, 404, 1, 0, 0, 0, 20, 411, 1, 0, 0, 0, 22, 420, 1, 0, 0, 0, 24, 427, 1, 0, 0, 0, 26, 437, 1, 0, 0, 0, 28, 444, 1, 0, 0, 0, 30, 451, 1, 0, 0, 0, 32, 465, 1, 0, 0, 0, 34, 472, 1, 0, 0, 0, 36, 480, 1, 0, 0, 0, 38, 489, 1, 0, 0, 0, 40, 496, 1, 0, 0, 0, 42, 506, 1, 0, 0, 0, 44, 518, 1, 0, 0, 0, 46, 527, 1, 0, 0, 0, 48, 533, 1, 0, 0, 0, 50, 540, 1, 0, 0, 0, 52, 547, 1, 0, 0, 0, 54, 555, 1, 0, 0, 0, 56, 564, 1, 0, 0, 0, 58, 570, 1, 0, 0, 0, 60, 587, 1, 0, 0, 0, 62, 603, 1, 0, 0, 0, 64, 612, 1, 0, 0, 0, 66, 615, 1, 0, 0, 0, 68, 619, 1, 0, 0, 0, 70, 624, 1, 0, 0, 0, 72, 629, 1, 0, 0, 0, 74, 633, 1, 0, 0, 0, 76, 637, 1, 0, 0, 0, 78, 641, 1, 0, 0, 0, 80, 645, 1, 0, 0, 0, 82, 647, 1, 0, 0, 0, 84, 649, 1, 0, 0, 0, 86, 652, 1, 0, 0, 0, 88, 654, 1, 0, 0, 0, 90, 663, 1, 0, 0, 0, 92, 665, 1, 0, 0, 0, 94, 670, 1, 0, 0, 0, 96, 672, 1, 0, 0, 0, 98, 677, 1, 0, 0, 0, 100, 708, 1, 0, 0, 0, 102, 711, 1, 0, 0, 0, 104, 757, 1, 0, 0, 0, 106, 759, 1, 0, 0, 0, 108, 762, 1, 0, 0, 0, 110, 766, 1, 0, 0, 0, 112, 770, 1, 0, 0, 0, 114, 772, 1, 0, 0, 0, 116, 775, 1, 0, 0, 0, 118, 777, 1, 0, 0, 0, 120, 782, 1, 0, 0, 0, 122, 784, 1, 0, 0, 0, 124, 790, 1, 0, 0, 0, 126, 796, 1, 0, 0, 0, 128, 801, 1, 0, 0, 0, 130, 803, 1, 0, 0, 0, 132, 806, 1, 0, 0, 0, 134, 809, 1, 0, 0, 0, 136, 814, 1, 0, 0, 0, 138, 818, 1, 0, 0, 0, 140, 823, 1, 0, 0, 0, 142, 829, 1, 0, 0, 0, 144, 832, 1, 0, 0, 0, 146, 834, 1, 0, 0, 0, 148, 840, 1, 0, 0, 0, 150, 842, 1, 0, 0, 0, 152, 847, 1, 0, 0, 0, 154, 850, 1, 0, 0, 0, 156, 853, 1, 0, 0, 0, 158, 856, 1, 0, 0, 0, 160, 858, 1, 0, 0, 0, 162, 861, 1, 0, 0, 0, 164, 863, 1, 0, 0, 0, 166, 866, 1, 0, 0, 0, 168, 868, 1, 0, 0, 0, 170, 870, 1, 0, 0, 0, 172, 872, 1, 0, 0, 0, 174, 874, 1, 0, 0, 0, 176, 876, 1, 0, 0, 0, 178, 881, 1, 0, 0, 0, 180, 902, 1, 0, 0, 0, 182, 904, 1, 0, 0, 0, 184, 912, 1, 0, 0, 0, 186, 914, 1, 0, 0, 0, 188, 918, 1, 0, 0, 0, 190, 922, 1, 0, 0, 0, 192, 926, 1, 0, 0, 0, 194, 931, 1, 0, 0, 0, 196, 935, 1, 0, 0, 0, 198, 939, 1, 0, 0, 0, 200, 943, 1, 0, 0, 0, 202, 947, 1, 0, 0, 0, 204, 951, 1, 0, 0, 0, 206, 960, 1, 0, 0, 0, 208, 964, 1, 0, 0, 0, 210, 968, 1, 0, 0, 0, 212, 972, 1, 0, 0, 0, 214, 976, 1, 0, 0, 0, 216, 981, 1, 0, 0, 0, 218, 985, 1, 0, 0, 0, 220, 993, 1, 0, 0, 0, 222, 1014, 1, 0, 0, 0, 224, 1018, 1, 0, 0, 0, 226, 1022, 1, 0, 0, 0, 228, 1026, 1, 0, 0, 0, 230, 1030, 1, 0, 0, 0, 232, 1034, 1, 0, 0, 0, 234, 1039, 1, 0, 0, 0, 236, 1043, 1, 0, 0, 0, 238, 1047, 1, 0, 0, 0, 240, 1051, 1, 0, 0, 0, 242, 1054, 1, 0, 0, 0, 244, 1058, 1, 0, 0, 0, 246, 1062, 1, 0, 0, 0, 248, 1066, 1, 0, 0, 0, 250, 1070, 1, 0, 0, 0, 252, 1075, 1, 0, 0, 0, 254, 1080, 1, 0, 0, 0, 256, 1085, 1, 0, 0, 0, 258, 1092, 1, 0, 0, 0, 260, 1101, 1, 0, 0, 0, 262, 1108, 1, 0, 0, 0, 264, 1112, 1, 0, 0, 0, 266, 1116, 1, 0, 0, 0, 268, 1120, 1, 0, 0, 0, 270, 1124, 1, 0, 0, 0, 272, 1128, 1, 0, 0, 0, 274, 1134, 1, 0, 0, 0, 276, 1138, 1, 0, 0, 0, 278, 1142, 1, 0, 0, 0, 280, 1146, 1, 0, 0, 0, 282, 1150, 1, 0, 0, 0, 284, 1154, 1, 0, 0, 0, 286, 1158, 1, 0, 0, 0, 288, 1162, 1, 0, 0, 0, 290, 1166, 1, 0, 0, 0, 292, 1170, 1, 0, 0, 0, 294, 1175, 1, 0, 0, 0, 296, 1179, 1, 0, 0, 0, 298, 1183, 1, 0, 0, 0, 300, 1188, 1, 0, 0, 0, 302, 1192, 1, 0, 0, 0, 304, 1196, 1, 0, 0, 0, 306, 1200, 1, 0, 0, 0, 308, 1204, 1, 0, 0, 0, 310, 1210, 1, 0, 0, 0, 312, 1214, 1, 0, 0, 0, 314, 1218, 1, 0, 0, 0, 316, 1222, 1, 0, 0, 0, 318, 1226, 1, 0, 0, 0, 320, 1230, 1, 0, 0, 0, 322, 1234, 1, 0, 0, 0, 324, 1239, 1, 0, 0, 0, 326, 1243, 1, 0, 0, 0, 328, 1247, 1, 0, 0, 0, 330, 1251, 1, 0, 0, 0, 332, 1255, 1, 0, 0, 0, 334, 1259, 1, 0, 0, 0, 336, 1263, 1, 0, 0, 0, 338, 1268, 1, 0, 0, 0, 340, 1273, 1, 0, 0, 0, 342, 1277, 1, 0, 0, 0, 344, 1281, 1, 0, 0, 0, 346, 1285, 1, 0, 0, 0, 348, 1290, 1, 0, 0, 0, 350, 1300, 1, 0, 0, 0, 352, 1304, 1, 0, 0, 0, 354, 1308, 1, 0, 0, 0, 356, 1312, 1, 0, 0, 0, 358, 1317, 1, 0, 0, 0, 360, 1324, 1, 0, 0, 0, 362, 1328, 1, 0, 0, 0, 364, 1332, 1, 0, 0, 0, 366, 1336, 1, 0, 0, 0, 368, 1340, 1, 0, 0, 0, 370, 1345, 1, 0, 0, 0, 372, 1351, 1, 0, 0, 0, 374, 1355, 1, 0, 0, 0, 376, 1359, 1, 0, 0, 0, 378, 1363, 1, 0, 0, 0, 380, 1369, 1, 0, 0, 0, 382, 1373, 1, 0, 0, 0, 384, 1377, 1, 0, 0, 0, 386, 1381, 1, 0, 0, 0, 388, 1387, 1, 0, 0, 0, 390, 1393, 1, 0, 0, 0, 392, 1399, 1, 0, 0, 0, 394, 395, 5, 100, 0, 0, 395, 396, 5, 105, 0, 0, 396, 397, 5, 115, 0, 0, 397, 398, 5, 115, 0, 0, 398, 399, 5, 101, 0, 0, 399, 400, 5, 99, 0, 0, 400, 401, 5, 116, 0, 0, 401, 402, 1, 0, 0, 0, 402, 403, 6, 0, 0, 0, 403, 17, 1, 0, 0, 0, 404, 405, 5, 100, 0, 0, 405, 406, 5, 114, 0, 0, 406, 407, 5, 111, 0, 0, 407, 408, 5, 112, 0, 0, 408, 409, 1, 0, 0, 0, 409, 410, 6, 1, 1, 0, 410, 19, 1, 0, 0, 0, 411, 412, 5, 101, 0, 0, 412, 413, 5, 110, 0, 0, 413, 414, 5, 114, 0, 0, 414, 415, 5, 105, 0, 0, 415, 416, 5, 99, 0, 0, 416, 417, 5, 104, 0, 0, 417, 418, 1, 0, 0, 0, 418, 419, 6, 2, 2, 0, 419, 21, 1, 0, 0, 0, 420, 421, 5, 101, 0, 0, 421, 422, 5, 118, 0, 0, 422, 423, 5, 97, 0, 0, 423, 424, 5, 108, 0, 0, 424, 425, 1, 0, 0, 0, 425, 426, 6, 3, 0, 0, 426, 23, 1, 0, 0, 0, 427, 428, 5, 101, 0, 0, 428, 429, 5, 120, 0, 0, 429, 430, 5, 112, 0, 0, 430, 431, 5, 108, 0, 0, 431, 432, 5, 97, 0, 0, 432, 433, 5, 105, 0, 0, 433, 434, 5, 110, 0, 0, 434, 435, 1, 0, 0, 0, 435, 436, 6, 4, 3, 0, 436, 25, 1, 0, 0, 0, 437, 438, 5, 102, 0, 0, 438, 439, 5, 114, 0, 0, 439, 440, 5, 111, 0, 0, 440, 441, 5, 109, 0, 0, 441, 442, 1, 0, 0, 0, 442, 443, 6, 5, 4, 0, 443, 27, 1, 0, 0, 0, 444, 445, 5, 103, 0, 0, 445, 446, 5, 114, 0, 0, 446, 447, 5, 111, 0, 0, 447, 448, 5, 107, 0, 0, 448, 449, 1, 0, 0, 0, 449, 450, 6, 6, 0, 0, 450, 29, 1, 0, 0, 0, 451, 452, 5, 105, 0, 0, 452, 453, 5, 110, 0, 0, 453, 454, 5, 108, 0, 0, 454, 455, 5, 105, 0, 0, 455, 456, 5, 110, 0, 0, 456, 457, 5, 101, 0, 0, 457, 458, 5, 115, 0, 0, 458, 459, 5, 116, 0, 0, 459, 460, 5, 97, 0, 0, 460, 461, 5, 116, 0, 0, 461, 462, 5, 115, 0, 0, 462, 463, 1, 0, 0, 0, 463, 464, 6, 7, 0, 0, 464, 31, 1, 0, 0, 0, 465, 466, 5, 107, 0, 0, 466, 467, 5, 101, 0, 0, 467, 468, 5, 101, 0, 0, 468, 469, 5, 112, 0, 0, 469, 470, 1, 0, 0, 0, 470, 471, 6, 8, 1, 0, 471, 33, 1, 0, 0, 0, 472, 473, 5, 108, 0, 0, 473, 474, 5, 105, 0, 0, 474, 475, 5, 109, 0, 0, 475, 476, 5, 105, 0, 0, 476, 477, 5, 116, 0, 0, 477, 478, 1, 0, 0, 0, 478, 479, 6, 9, 0, 0, 479, 35, 1, 0, 0, 0, 480, 481, 5, 108, 0, 0, 481, 482, 5, 111, 0, 0, 482, 483, 5, 111, 0, 0, 483, 484, 5, 107, 0, 0, 484, 485, 5, 117, 0, 0, 485, 486, 5, 112, 0, 0, 486, 487, 1, 0, 0, 0, 487, 488, 6, 10, 5, 0, 488, 37, 1, 0, 0, 0, 489, 490, 5, 109, 0, 0, 490, 491, 5, 101, 0, 0, 491, 492, 5, 116, 0, 0, 492, 493, 5, 97, 0, 0, 493, 494, 1, 0, 0, 0, 494, 495, 6, 11, 6, 0, 495, 39, 1, 0, 0, 0, 496, 497, 5, 109, 0, 0, 497, 498, 5, 101, 0, 0, 498, 499, 5, 116, 0, 0, 499, 500, 5, 114, 0, 0, 500, 501, 5, 105, 0, 0, 501, 502, 5, 99, 0, 0, 502, 503, 5, 115, 0, 0, 503, 504, 1, 0, 0, 0, 504, 505, 6, 12, 7, 0, 505, 41, 1, 0, 0, 0, 506, 507, 5, 109, 0, 0, 507, 508, 5, 118, 0, 0, 508, 509, 5, 95, 0, 0, 509, 510, 5, 101, 0, 0, 510, 511, 5, 120, 0, 0, 511, 512, 5, 112, 0, 0, 512, 513, 5, 97, 0, 0, 513, 514, 5, 110, 0, 0, 514, 515, 5, 100, 0, 0, 515, 516, 1, 0, 0, 0, 516, 517, 6, 13, 8, 0, 517, 43, 1, 0, 0, 0, 518, 519, 5, 114, 0, 0, 519, 520, 5, 101, 0, 0, 520, 521, 5, 110, 0, 0, 521, 522, 5, 97, 0, 0, 522, 523, 5, 109, 0, 0, 523, 524, 5, 101, 0, 0, 524, 525, 1, 0, 0, 0, 525, 526, 6, 14, 9, 0, 526, 45, 1, 0, 0, 0, 527, 528, 5, 114, 0, 0, 528, 529, 5, 111, 0, 0, 529, 530, 5, 119, 0, 0, 530, 531, 1, 0, 0, 0, 531, 532, 6, 15, 0, 0, 532, 47, 1, 0, 0, 0, 533, 534, 5, 115, 0, 0, 534, 535, 5, 104, 0, 0, 535, 536, 5, 111, 0, 0, 536, 537, 5, 119, 0, 0, 537, 538, 1, 0, 0, 0, 538, 539, 6, 16, 10, 0, 539, 49, 1, 0, 0, 0, 540, 541, 5, 115, 0, 0, 541, 542, 5, 111, 0, 0, 542, 543, 5, 114, 0, 0, 543, 544, 5, 116, 0, 0, 544, 545, 1, 0, 0, 0, 545, 546, 6, 17, 0, 0, 546, 51, 1, 0, 0, 0, 547, 548, 5, 115, 0, 0, 548, 549, 5, 116, 0, 0, 549, 550, 5, 97, 0, 0, 550, 551, 5, 116, 0, 0, 551, 552, 5, 115, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 6, 18, 0, 0, 554, 53, 1, 0, 0, 0, 555, 556, 5, 119, 0, 0, 556, 557, 5, 104, 0, 0, 557, 558, 5, 101, 0, 0, 558, 559, 5, 114, 0, 0, 559, 560, 5, 101, 0, 0, 560, 561, 1, 0, 0, 0, 561, 562, 6, 19, 0, 0, 562, 55, 1, 0, 0, 0, 563, 565, 8, 0, 0, 0, 564, 563, 1, 0, 0, 0, 565, 566, 1, 0, 0, 0, 566, 564, 1, 0, 0, 0, 566, 567, 1, 0, 0, 0, 567, 568, 1, 0, 0, 0, 568, 569, 6, 20, 0, 0, 569, 57, 1, 0, 0, 0, 570, 571, 5, 47, 0, 0, 571, 572, 5, 47, 0, 0, 572, 576, 1, 0, 0, 0, 573, 575, 8, 1, 0, 0, 574, 573, 1, 0, 0, 0, 575, 578, 1, 0, 0, 0, 576, 574, 1, 0, 0, 0, 576, 577, 1, 0, 0, 0, 577, 580, 1, 0, 0, 0, 578, 576, 1, 0, 0, 0, 579, 581, 5, 13, 0, 0, 580, 579, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 583, 1, 0, 0, 0, 582, 584, 5, 10, 0, 0, 583, 582, 1, 0, 0, 0, 583, 584, 1, 0, 0, 0, 584, 585, 1, 0, 0, 0, 585, 586, 6, 21, 11, 0, 586, 59, 1, 0, 0, 0, 587, 588, 5, 47, 0, 0, 588, 589, 5, 42, 0, 0, 589, 594, 1, 0, 0, 0, 590, 593, 3, 60, 22, 0, 591, 593, 9, 0, 0, 0, 592, 590, 1, 0, 0, 0, 592, 591, 1, 0, 0, 0, 593, 596, 1, 0, 0, 0, 594, 595, 1, 0, 0, 0, 594, 592, 1, 0, 0, 0, 595, 597, 1, 0, 0, 0, 596, 594, 1, 0, 0, 0, 597, 598, 5, 42, 0, 0, 598, 599, 5, 47, 0, 0, 599, 600, 1, 0, 0, 0, 600, 601, 6, 22, 11, 0, 601, 61, 1, 0, 0, 0, 602, 604, 7, 2, 0, 0, 603, 602, 1, 0, 0, 0, 604, 605, 1, 0, 0, 0, 605, 603, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 1, 0, 0, 0, 607, 608, 6, 23, 11, 0, 608, 63, 1, 0, 0, 0, 609, 613, 8, 3, 0, 0, 610, 611, 5, 47, 0, 0, 611, 613, 8, 4, 0, 0, 612, 609, 1, 0, 0, 0, 612, 610, 1, 0, 0, 0, 613, 65, 1, 0, 0, 0, 614, 616, 3, 64, 24, 0, 615, 614, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 617, 615, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 67, 1, 0, 0, 0, 619, 620, 3, 176, 80, 0, 620, 621, 1, 0, 0, 0, 621, 622, 6, 26, 12, 0, 622, 623, 6, 26, 13, 0, 623, 69, 1, 0, 0, 0, 624, 625, 3, 78, 31, 0, 625, 626, 1, 0, 0, 0, 626, 627, 6, 27, 14, 0, 627, 628, 6, 27, 15, 0, 628, 71, 1, 0, 0, 0, 629, 630, 3, 62, 23, 0, 630, 631, 1, 0, 0, 0, 631, 632, 6, 28, 11, 0, 632, 73, 1, 0, 0, 0, 633, 634, 3, 58, 21, 0, 634, 635, 1, 0, 0, 0, 635, 636, 6, 29, 11, 0, 636, 75, 1, 0, 0, 0, 637, 638, 3, 60, 22, 0, 638, 639, 1, 0, 0, 0, 639, 640, 6, 30, 11, 0, 640, 77, 1, 0, 0, 0, 641, 642, 5, 124, 0, 0, 642, 643, 1, 0, 0, 0, 643, 644, 6, 31, 15, 0, 644, 79, 1, 0, 0, 0, 645, 646, 7, 5, 0, 0, 646, 81, 1, 0, 0, 0, 647, 648, 7, 6, 0, 0, 648, 83, 1, 0, 0, 0, 649, 650, 5, 92, 0, 0, 650, 651, 7, 7, 0, 0, 651, 85, 1, 0, 0, 0, 652, 653, 8, 8, 0, 0, 653, 87, 1, 0, 0, 0, 654, 656, 7, 9, 0, 0, 655, 657, 7, 10, 0, 0, 656, 655, 1, 0, 0, 0, 656, 657, 1, 0, 0, 0, 657, 659, 1, 0, 0, 0, 658, 660, 3, 80, 32, 0, 659, 658, 1, 0, 0, 0, 660, 661, 1, 0, 0, 0, 661, 659, 1, 0, 0, 0, 661, 662, 1, 0, 0, 0, 662, 89, 1, 0, 0, 0, 663, 664, 5, 64, 0, 0, 664, 91, 1, 0, 0, 0, 665, 666, 5, 96, 0, 0, 666, 93, 1, 0, 0, 0, 667, 671, 8, 11, 0, 0, 668, 669, 5, 96, 0, 0, 669, 671, 5, 96, 0, 0, 670, 667, 1, 0, 0, 0, 670, 668, 1, 0, 0, 0, 671, 95, 1, 0, 0, 0, 672, 673, 5, 95, 0, 0, 673, 97, 1, 0, 0, 0, 674, 678, 3, 82, 33, 0, 675, 678, 3, 80, 32, 0, 676, 678, 3, 96, 40, 0, 677, 674, 1, 0, 0, 0, 677, 675, 1, 0, 0, 0, 677, 676, 1, 0, 0, 0, 678, 99, 1, 0, 0, 0, 679, 684, 5, 34, 0, 0, 680, 683, 3, 84, 34, 0, 681, 683, 3, 86, 35, 0, 682, 680, 1, 0, 0, 0, 682, 681, 1, 0, 0, 0, 683, 686, 1, 0, 0, 0, 684, 682, 1, 0, 0, 0, 684, 685, 1, 0, 0, 0, 685, 687, 1, 0, 0, 0, 686, 684, 1, 0, 0, 0, 687, 709, 5, 34, 0, 0, 688, 689, 5, 34, 0, 0, 689, 690, 5, 34, 0, 0, 690, 691, 5, 34, 0, 0, 691, 695, 1, 0, 0, 0, 692, 694, 8, 1, 0, 0, 693, 692, 1, 0, 0, 0, 694, 697, 1, 0, 0, 0, 695, 696, 1, 0, 0, 0, 695, 693, 1, 0, 0, 0, 696, 698, 1, 0, 0, 0, 697, 695, 1, 0, 0, 0, 698, 699, 5, 34, 0, 0, 699, 700, 5, 34, 0, 0, 700, 701, 5, 34, 0, 0, 701, 703, 1, 0, 0, 0, 702, 704, 5, 34, 0, 0, 703, 702, 1, 0, 0, 0, 703, 704, 1, 0, 0, 0, 704, 706, 1, 0, 0, 0, 705, 707, 5, 34, 0, 0, 706, 705, 1, 0, 0, 0, 706, 707, 1, 0, 0, 0, 707, 709, 1, 0, 0, 0, 708, 679, 1, 0, 0, 0, 708, 688, 1, 0, 0, 0, 709, 101, 1, 0, 0, 0, 710, 712, 3, 80, 32, 0, 711, 710, 1, 0, 0, 0, 712, 713, 1, 0, 0, 0, 713, 711, 1, 0, 0, 0, 713, 714, 1, 0, 0, 0, 714, 103, 1, 0, 0, 0, 715, 717, 3, 80, 32, 0, 716, 715, 1, 0, 0, 0, 717, 718, 1, 0, 0, 0, 718, 716, 1, 0, 0, 0, 718, 719, 1, 0, 0, 0, 719, 720, 1, 0, 0, 0, 720, 724, 3, 120, 52, 0, 721, 723, 3, 80, 32, 0, 722, 721, 1, 0, 0, 0, 723, 726, 1, 0, 0, 0, 724, 722, 1, 0, 0, 0, 724, 725, 1, 0, 0, 0, 725, 758, 1, 0, 0, 0, 726, 724, 1, 0, 0, 0, 727, 729, 3, 120, 52, 0, 728, 730, 3, 80, 32, 0, 729, 728, 1, 0, 0, 0, 730, 731, 1, 0, 0, 0, 731, 729, 1, 0, 0, 0, 731, 732, 1, 0, 0, 0, 732, 758, 1, 0, 0, 0, 733, 735, 3, 80, 32, 0, 734, 733, 1, 0, 0, 0, 735, 736, 1, 0, 0, 0, 736, 734, 1, 0, 0, 0, 736, 737, 1, 0, 0, 0, 737, 745, 1, 0, 0, 0, 738, 742, 3, 120, 52, 0, 739, 741, 3, 80, 32, 0, 740, 739, 1, 0, 0, 0, 741, 744, 1, 0, 0, 0, 742, 740, 1, 0, 0, 0, 742, 743, 1, 0, 0, 0, 743, 746, 1, 0, 0, 0, 744, 742, 1, 0, 0, 0, 745, 738, 1, 0, 0, 0, 745, 746, 1, 0, 0, 0, 746, 747, 1, 0, 0, 0, 747, 748, 3, 88, 36, 0, 748, 758, 1, 0, 0, 0, 749, 751, 3, 120, 52, 0, 750, 752, 3, 80, 32, 0, 751, 750, 1, 0, 0, 0, 752, 753, 1, 0, 0, 0, 753, 751, 1, 0, 0, 0, 753, 754, 1, 0, 0, 0, 754, 755, 1, 0, 0, 0, 755, 756, 3, 88, 36, 0, 756, 758, 1, 0, 0, 0, 757, 716, 1, 0, 0, 0, 757, 727, 1, 0, 0, 0, 757, 734, 1, 0, 0, 0, 757, 749, 1, 0, 0, 0, 758, 105, 1, 0, 0, 0, 759, 760, 5, 98, 0, 0, 760, 761, 5, 121, 0, 0, 761, 107, 1, 0, 0, 0, 762, 763, 5, 97, 0, 0, 763, 764, 5, 110, 0, 0, 764, 765, 5, 100, 0, 0, 765, 109, 1, 0, 0, 0, 766, 767, 5, 97, 0, 0, 767, 768, 5, 115, 0, 0, 768, 769, 5, 99, 0, 0, 769, 111, 1, 0, 0, 0, 770, 771, 5, 61, 0, 0, 771, 113, 1, 0, 0, 0, 772, 773, 5, 58, 0, 0, 773, 774, 5, 58, 0, 0, 774, 115, 1, 0, 0, 0, 775, 776, 5, 44, 0, 0, 776, 117, 1, 0, 0, 0, 777, 778, 5, 100, 0, 0, 778, 779, 5, 101, 0, 0, 779, 780, 5, 115, 0, 0, 780, 781, 5, 99, 0, 0, 781, 119, 1, 0, 0, 0, 782, 783, 5, 46, 0, 0, 783, 121, 1, 0, 0, 0, 784, 785, 5, 102, 0, 0, 785, 786, 5, 97, 0, 0, 786, 787, 5, 108, 0, 0, 787, 788, 5, 115, 0, 0, 788, 789, 5, 101, 0, 0, 789, 123, 1, 0, 0, 0, 790, 791, 5, 102, 0, 0, 791, 792, 5, 105, 0, 0, 792, 793, 5, 114, 0, 0, 793, 794, 5, 115, 0, 0, 794, 795, 5, 116, 0, 0, 795, 125, 1, 0, 0, 0, 796, 797, 5, 108, 0, 0, 797, 798, 5, 97, 0, 0, 798, 799, 5, 115, 0, 0, 799, 800, 5, 116, 0, 0, 800, 127, 1, 0, 0, 0, 801, 802, 5, 40, 0, 0, 802, 129, 1, 0, 0, 0, 803, 804, 5, 105, 0, 0, 804, 805, 5, 110, 0, 0, 805, 131, 1, 0, 0, 0, 806, 807, 5, 105, 0, 0, 807, 808, 5, 115, 0, 0, 808, 133, 1, 0, 0, 0, 809, 810, 5, 108, 0, 0, 810, 811, 5, 105, 0, 0, 811, 812, 5, 107, 0, 0, 812, 813, 5, 101, 0, 0, 813, 135, 1, 0, 0, 0, 814, 815, 5, 110, 0, 0, 815, 816, 5, 111, 0, 0, 816, 817, 5, 116, 0, 0, 817, 137, 1, 0, 0, 0, 818, 819, 5, 110, 0, 0, 819, 820, 5, 117, 0, 0, 820, 821, 5, 108, 0, 0, 821, 822, 5, 108, 0, 0, 822, 139, 1, 0, 0, 0, 823, 824, 5, 110, 0, 0, 824, 825, 5, 117, 0, 0, 825, 826, 5, 108, 0, 0, 826, 827, 5, 108, 0, 0, 827, 828, 5, 115, 0, 0, 828, 141, 1, 0, 0, 0, 829, 830, 5, 111, 0, 0, 830, 831, 5, 114, 0, 0, 831, 143, 1, 0, 0, 0, 832, 833, 5, 63, 0, 0, 833, 145, 1, 0, 0, 0, 834, 835, 5, 114, 0, 0, 835, 836, 5, 108, 0, 0, 836, 837, 5, 105, 0, 0, 837, 838, 5, 107, 0, 0, 838, 839, 5, 101, 0, 0, 839, 147, 1, 0, 0, 0, 840, 841, 5, 41, 0, 0, 841, 149, 1, 0, 0, 0, 842, 843, 5, 116, 0, 0, 843, 844, 5, 114, 0, 0, 844, 845, 5, 117, 0, 0, 845, 846, 5, 101, 0, 0, 846, 151, 1, 0, 0, 0, 847, 848, 5, 61, 0, 0, 848, 849, 5, 61, 0, 0, 849, 153, 1, 0, 0, 0, 850, 851, 5, 61, 0, 0, 851, 852, 5, 126, 0, 0, 852, 155, 1, 0, 0, 0, 853, 854, 5, 33, 0, 0, 854, 855, 5, 61, 0, 0, 855, 157, 1, 0, 0, 0, 856, 857, 5, 60, 0, 0, 857, 159, 1, 0, 0, 0, 858, 859, 5, 60, 0, 0, 859, 860, 5, 61, 0, 0, 860, 161, 1, 0, 0, 0, 861, 862, 5, 62, 0, 0, 862, 163, 1, 0, 0, 0, 863, 864, 5, 62, 0, 0, 864, 865, 5, 61, 0, 0, 865, 165, 1, 0, 0, 0, 866, 867, 5, 43, 0, 0, 867, 167, 1, 0, 0, 0, 868, 869, 5, 45, 0, 0, 869, 169, 1, 0, 0, 0, 870, 871, 5, 42, 0, 0, 871, 171, 1, 0, 0, 0, 872, 873, 5, 47, 0, 0, 873, 173, 1, 0, 0, 0, 874, 875, 5, 37, 0, 0, 875, 175, 1, 0, 0, 0, 876, 877, 5, 91, 0, 0, 877, 878, 1, 0, 0, 0, 878, 879, 6, 80, 0, 0, 879, 880, 6, 80, 0, 0, 880, 177, 1, 0, 0, 0, 881, 882, 5, 93, 0, 0, 882, 883, 1, 0, 0, 0, 883, 884, 6, 81, 15, 0, 884, 885, 6, 81, 15, 0, 885, 179, 1, 0, 0, 0, 886, 890, 3, 82, 33, 0, 887, 889, 3, 98, 41, 0, 888, 887, 1, 0, 0, 0, 889, 892, 1, 0, 0, 0, 890, 888, 1, 0, 0, 0, 890, 891, 1, 0, 0, 0, 891, 903, 1, 0, 0, 0, 892, 890, 1, 0, 0, 0, 893, 896, 3, 96, 40, 0, 894, 896, 3, 90, 37, 0, 895, 893, 1, 0, 0, 0, 895, 894, 1, 0, 0, 0, 896, 898, 1, 0, 0, 0, 897, 899, 3, 98, 41, 0, 898, 897, 1, 0, 0, 0, 899, 900, 1, 0, 0, 0, 900, 898, 1, 0, 0, 0, 900, 901, 1, 0, 0, 0, 901, 903, 1, 0, 0, 0, 902, 886, 1, 0, 0, 0, 902, 895, 1, 0, 0, 0, 903, 181, 1, 0, 0, 0, 904, 906, 3, 92, 38, 0, 905, 907, 3, 94, 39, 0, 906, 905, 1, 0, 0, 0, 907, 908, 1, 0, 0, 0, 908, 906, 1, 0, 0, 0, 908, 909, 1, 0, 0, 0, 909, 910, 1, 0, 0, 0, 910, 911, 3, 92, 38, 0, 911, 183, 1, 0, 0, 0, 912, 913, 3, 182, 83, 0, 913, 185, 1, 0, 0, 0, 914, 915, 3, 58, 21, 0, 915, 916, 1, 0, 0, 0, 916, 917, 6, 85, 11, 0, 917, 187, 1, 0, 0, 0, 918, 919, 3, 60, 22, 0, 919, 920, 1, 0, 0, 0, 920, 921, 6, 86, 11, 0, 921, 189, 1, 0, 0, 0, 922, 923, 3, 62, 23, 0, 923, 924, 1, 0, 0, 0, 924, 925, 6, 87, 11, 0, 925, 191, 1, 0, 0, 0, 926, 927, 3, 78, 31, 0, 927, 928, 1, 0, 0, 0, 928, 929, 6, 88, 14, 0, 929, 930, 6, 88, 15, 0, 930, 193, 1, 0, 0, 0, 931, 932, 3, 176, 80, 0, 932, 933, 1, 0, 0, 0, 933, 934, 6, 89, 12, 0, 934, 195, 1, 0, 0, 0, 935, 936, 3, 178, 81, 0, 936, 937, 1, 0, 0, 0, 937, 938, 6, 90, 16, 0, 938, 197, 1, 0, 0, 0, 939, 940, 3, 116, 50, 0, 940, 941, 1, 0, 0, 0, 941, 942, 6, 91, 17, 0, 942, 199, 1, 0, 0, 0, 943, 944, 3, 112, 48, 0, 944, 945, 1, 0, 0, 0, 945, 946, 6, 92, 18, 0, 946, 201, 1, 0, 0, 0, 947, 948, 3, 100, 42, 0, 948, 949, 1, 0, 0, 0, 949, 950, 6, 93, 19, 0, 950, 203, 1, 0, 0, 0, 951, 952, 5, 109, 0, 0, 952, 953, 5, 101, 0, 0, 953, 954, 5, 116, 0, 0, 954, 955, 5, 97, 0, 0, 955, 956, 5, 100, 0, 0, 956, 957, 5, 97, 0, 0, 957, 958, 5, 116, 0, 0, 958, 959, 5, 97, 0, 0, 959, 205, 1, 0, 0, 0, 960, 961, 3, 66, 25, 0, 961, 962, 1, 0, 0, 0, 962, 963, 6, 95, 20, 0, 963, 207, 1, 0, 0, 0, 964, 965, 3, 58, 21, 0, 965, 966, 1, 0, 0, 0, 966, 967, 6, 96, 11, 0, 967, 209, 1, 0, 0, 0, 968, 969, 3, 60, 22, 0, 969, 970, 1, 0, 0, 0, 970, 971, 6, 97, 11, 0, 971, 211, 1, 0, 0, 0, 972, 973, 3, 62, 23, 0, 973, 974, 1, 0, 0, 0, 974, 975, 6, 98, 11, 0, 975, 213, 1, 0, 0, 0, 976, 977, 3, 78, 31, 0, 977, 978, 1, 0, 0, 0, 978, 979, 6, 99, 14, 0, 979, 980, 6, 99, 15, 0, 980, 215, 1, 0, 0, 0, 981, 982, 3, 120, 52, 0, 982, 983, 1, 0, 0, 0, 983, 984, 6, 100, 21, 0, 984, 217, 1, 0, 0, 0, 985, 986, 3, 116, 50, 0, 986, 987, 1, 0, 0, 0, 987, 988, 6, 101, 17, 0, 988, 219, 1, 0, 0, 0, 989, 994, 3, 82, 33, 0, 990, 994, 3, 80, 32, 0, 991, 994, 3, 96, 40, 0, 992, 994, 3, 170, 77, 0, 993, 989, 1, 0, 0, 0, 993, 990, 1, 0, 0, 0, 993, 991, 1, 0, 0, 0, 993, 992, 1, 0, 0, 0, 994, 221, 1, 0, 0, 0, 995, 998, 3, 82, 33, 0, 996, 998, 3, 170, 77, 0, 997, 995, 1, 0, 0, 0, 997, 996, 1, 0, 0, 0, 998, 1002, 1, 0, 0, 0, 999, 1001, 3, 220, 102, 0, 1000, 999, 1, 0, 0, 0, 1001, 1004, 1, 0, 0, 0, 1002, 1000, 1, 0, 0, 0, 1002, 1003, 1, 0, 0, 0, 1003, 1015, 1, 0, 0, 0, 1004, 1002, 1, 0, 0, 0, 1005, 1008, 3, 96, 40, 0, 1006, 1008, 3, 90, 37, 0, 1007, 1005, 1, 0, 0, 0, 1007, 1006, 1, 0, 0, 0, 1008, 1010, 1, 0, 0, 0, 1009, 1011, 3, 220, 102, 0, 1010, 1009, 1, 0, 0, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1010, 1, 0, 0, 0, 1012, 1013, 1, 0, 0, 0, 1013, 1015, 1, 0, 0, 0, 1014, 997, 1, 0, 0, 0, 1014, 1007, 1, 0, 0, 0, 1015, 223, 1, 0, 0, 0, 1016, 1019, 3, 222, 103, 0, 1017, 1019, 3, 182, 83, 0, 1018, 1016, 1, 0, 0, 0, 1018, 1017, 1, 0, 0, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1018, 1, 0, 0, 0, 1020, 1021, 1, 0, 0, 0, 1021, 225, 1, 0, 0, 0, 1022, 1023, 3, 58, 21, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 105, 11, 0, 1025, 227, 1, 0, 0, 0, 1026, 1027, 3, 60, 22, 0, 1027, 1028, 1, 0, 0, 0, 1028, 1029, 6, 106, 11, 0, 1029, 229, 1, 0, 0, 0, 1030, 1031, 3, 62, 23, 0, 1031, 1032, 1, 0, 0, 0, 1032, 1033, 6, 107, 11, 0, 1033, 231, 1, 0, 0, 0, 1034, 1035, 3, 78, 31, 0, 1035, 1036, 1, 0, 0, 0, 1036, 1037, 6, 108, 14, 0, 1037, 1038, 6, 108, 15, 0, 1038, 233, 1, 0, 0, 0, 1039, 1040, 3, 112, 48, 0, 1040, 1041, 1, 0, 0, 0, 1041, 1042, 6, 109, 18, 0, 1042, 235, 1, 0, 0, 0, 1043, 1044, 3, 116, 50, 0, 1044, 1045, 1, 0, 0, 0, 1045, 1046, 6, 110, 17, 0, 1046, 237, 1, 0, 0, 0, 1047, 1048, 3, 120, 52, 0, 1048, 1049, 1, 0, 0, 0, 1049, 1050, 6, 111, 21, 0, 1050, 239, 1, 0, 0, 0, 1051, 1052, 5, 97, 0, 0, 1052, 1053, 5, 115, 0, 0, 1053, 241, 1, 0, 0, 0, 1054, 1055, 3, 224, 104, 0, 1055, 1056, 1, 0, 0, 0, 1056, 1057, 6, 113, 22, 0, 1057, 243, 1, 0, 0, 0, 1058, 1059, 3, 58, 21, 0, 1059, 1060, 1, 0, 0, 0, 1060, 1061, 6, 114, 11, 0, 1061, 245, 1, 0, 0, 0, 1062, 1063, 3, 60, 22, 0, 1063, 1064, 1, 0, 0, 0, 1064, 1065, 6, 115, 11, 0, 1065, 247, 1, 0, 0, 0, 1066, 1067, 3, 62, 23, 0, 1067, 1068, 1, 0, 0, 0, 1068, 1069, 6, 116, 11, 0, 1069, 249, 1, 0, 0, 0, 1070, 1071, 3, 78, 31, 0, 1071, 1072, 1, 0, 0, 0, 1072, 1073, 6, 117, 14, 0, 1073, 1074, 6, 117, 15, 0, 1074, 251, 1, 0, 0, 0, 1075, 1076, 3, 176, 80, 0, 1076, 1077, 1, 0, 0, 0, 1077, 1078, 6, 118, 12, 0, 1078, 1079, 6, 118, 23, 0, 1079, 253, 1, 0, 0, 0, 1080, 1081, 5, 111, 0, 0, 1081, 1082, 5, 110, 0, 0, 1082, 1083, 1, 0, 0, 0, 1083, 1084, 6, 119, 24, 0, 1084, 255, 1, 0, 0, 0, 1085, 1086, 5, 119, 0, 0, 1086, 1087, 5, 105, 0, 0, 1087, 1088, 5, 116, 0, 0, 1088, 1089, 5, 104, 0, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 120, 24, 0, 1091, 257, 1, 0, 0, 0, 1092, 1093, 8, 12, 0, 0, 1093, 259, 1, 0, 0, 0, 1094, 1096, 3, 258, 121, 0, 1095, 1094, 1, 0, 0, 0, 1096, 1097, 1, 0, 0, 0, 1097, 1095, 1, 0, 0, 0, 1097, 1098, 1, 0, 0, 0, 1098, 1099, 1, 0, 0, 0, 1099, 1100, 3, 358, 171, 0, 1100, 1102, 1, 0, 0, 0, 1101, 1095, 1, 0, 0, 0, 1101, 1102, 1, 0, 0, 0, 1102, 1104, 1, 0, 0, 0, 1103, 1105, 3, 258, 121, 0, 1104, 1103, 1, 0, 0, 0, 1105, 1106, 1, 0, 0, 0, 1106, 1104, 1, 0, 0, 0, 1106, 1107, 1, 0, 0, 0, 1107, 261, 1, 0, 0, 0, 1108, 1109, 3, 184, 84, 0, 1109, 1110, 1, 0, 0, 0, 1110, 1111, 6, 123, 25, 0, 1111, 263, 1, 0, 0, 0, 1112, 1113, 3, 260, 122, 0, 1113, 1114, 1, 0, 0, 0, 1114, 1115, 6, 124, 26, 0, 1115, 265, 1, 0, 0, 0, 1116, 1117, 3, 58, 21, 0, 1117, 1118, 1, 0, 0, 0, 1118, 1119, 6, 125, 11, 0, 1119, 267, 1, 0, 0, 0, 1120, 1121, 3, 60, 22, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 126, 11, 0, 1123, 269, 1, 0, 0, 0, 1124, 1125, 3, 62, 23, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 127, 11, 0, 1127, 271, 1, 0, 0, 0, 1128, 1129, 3, 78, 31, 0, 1129, 1130, 1, 0, 0, 0, 1130, 1131, 6, 128, 14, 0, 1131, 1132, 6, 128, 15, 0, 1132, 1133, 6, 128, 15, 0, 1133, 273, 1, 0, 0, 0, 1134, 1135, 3, 112, 48, 0, 1135, 1136, 1, 0, 0, 0, 1136, 1137, 6, 129, 18, 0, 1137, 275, 1, 0, 0, 0, 1138, 1139, 3, 116, 50, 0, 1139, 1140, 1, 0, 0, 0, 1140, 1141, 6, 130, 17, 0, 1141, 277, 1, 0, 0, 0, 1142, 1143, 3, 120, 52, 0, 1143, 1144, 1, 0, 0, 0, 1144, 1145, 6, 131, 21, 0, 1145, 279, 1, 0, 0, 0, 1146, 1147, 3, 256, 120, 0, 1147, 1148, 1, 0, 0, 0, 1148, 1149, 6, 132, 27, 0, 1149, 281, 1, 0, 0, 0, 1150, 1151, 3, 224, 104, 0, 1151, 1152, 1, 0, 0, 0, 1152, 1153, 6, 133, 22, 0, 1153, 283, 1, 0, 0, 0, 1154, 1155, 3, 184, 84, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1157, 6, 134, 25, 0, 1157, 285, 1, 0, 0, 0, 1158, 1159, 3, 58, 21, 0, 1159, 1160, 1, 0, 0, 0, 1160, 1161, 6, 135, 11, 0, 1161, 287, 1, 0, 0, 0, 1162, 1163, 3, 60, 22, 0, 1163, 1164, 1, 0, 0, 0, 1164, 1165, 6, 136, 11, 0, 1165, 289, 1, 0, 0, 0, 1166, 1167, 3, 62, 23, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1169, 6, 137, 11, 0, 1169, 291, 1, 0, 0, 0, 1170, 1171, 3, 78, 31, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1173, 6, 138, 14, 0, 1173, 1174, 6, 138, 15, 0, 1174, 293, 1, 0, 0, 0, 1175, 1176, 3, 116, 50, 0, 1176, 1177, 1, 0, 0, 0, 1177, 1178, 6, 139, 17, 0, 1178, 295, 1, 0, 0, 0, 1179, 1180, 3, 120, 52, 0, 1180, 1181, 1, 0, 0, 0, 1181, 1182, 6, 140, 21, 0, 1182, 297, 1, 0, 0, 0, 1183, 1184, 3, 254, 119, 0, 1184, 1185, 1, 0, 0, 0, 1185, 1186, 6, 141, 28, 0, 1186, 1187, 6, 141, 29, 0, 1187, 299, 1, 0, 0, 0, 1188, 1189, 3, 66, 25, 0, 1189, 1190, 1, 0, 0, 0, 1190, 1191, 6, 142, 20, 0, 1191, 301, 1, 0, 0, 0, 1192, 1193, 3, 58, 21, 0, 1193, 1194, 1, 0, 0, 0, 1194, 1195, 6, 143, 11, 0, 1195, 303, 1, 0, 0, 0, 1196, 1197, 3, 60, 22, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 144, 11, 0, 1199, 305, 1, 0, 0, 0, 1200, 1201, 3, 62, 23, 0, 1201, 1202, 1, 0, 0, 0, 1202, 1203, 6, 145, 11, 0, 1203, 307, 1, 0, 0, 0, 1204, 1205, 3, 78, 31, 0, 1205, 1206, 1, 0, 0, 0, 1206, 1207, 6, 146, 14, 0, 1207, 1208, 6, 146, 15, 0, 1208, 1209, 6, 146, 15, 0, 1209, 309, 1, 0, 0, 0, 1210, 1211, 3, 116, 50, 0, 1211, 1212, 1, 0, 0, 0, 1212, 1213, 6, 147, 17, 0, 1213, 311, 1, 0, 0, 0, 1214, 1215, 3, 120, 52, 0, 1215, 1216, 1, 0, 0, 0, 1216, 1217, 6, 148, 21, 0, 1217, 313, 1, 0, 0, 0, 1218, 1219, 3, 224, 104, 0, 1219, 1220, 1, 0, 0, 0, 1220, 1221, 6, 149, 22, 0, 1221, 315, 1, 0, 0, 0, 1222, 1223, 3, 58, 21, 0, 1223, 1224, 1, 0, 0, 0, 1224, 1225, 6, 150, 11, 0, 1225, 317, 1, 0, 0, 0, 1226, 1227, 3, 60, 22, 0, 1227, 1228, 1, 0, 0, 0, 1228, 1229, 6, 151, 11, 0, 1229, 319, 1, 0, 0, 0, 1230, 1231, 3, 62, 23, 0, 1231, 1232, 1, 0, 0, 0, 1232, 1233, 6, 152, 11, 0, 1233, 321, 1, 0, 0, 0, 1234, 1235, 3, 78, 31, 0, 1235, 1236, 1, 0, 0, 0, 1236, 1237, 6, 153, 14, 0, 1237, 1238, 6, 153, 15, 0, 1238, 323, 1, 0, 0, 0, 1239, 1240, 3, 120, 52, 0, 1240, 1241, 1, 0, 0, 0, 1241, 1242, 6, 154, 21, 0, 1242, 325, 1, 0, 0, 0, 1243, 1244, 3, 184, 84, 0, 1244, 1245, 1, 0, 0, 0, 1245, 1246, 6, 155, 25, 0, 1246, 327, 1, 0, 0, 0, 1247, 1248, 3, 180, 82, 0, 1248, 1249, 1, 0, 0, 0, 1249, 1250, 6, 156, 30, 0, 1250, 329, 1, 0, 0, 0, 1251, 1252, 3, 58, 21, 0, 1252, 1253, 1, 0, 0, 0, 1253, 1254, 6, 157, 11, 0, 1254, 331, 1, 0, 0, 0, 1255, 1256, 3, 60, 22, 0, 1256, 1257, 1, 0, 0, 0, 1257, 1258, 6, 158, 11, 0, 1258, 333, 1, 0, 0, 0, 1259, 1260, 3, 62, 23, 0, 1260, 1261, 1, 0, 0, 0, 1261, 1262, 6, 159, 11, 0, 1262, 335, 1, 0, 0, 0, 1263, 1264, 3, 78, 31, 0, 1264, 1265, 1, 0, 0, 0, 1265, 1266, 6, 160, 14, 0, 1266, 1267, 6, 160, 15, 0, 1267, 337, 1, 0, 0, 0, 1268, 1269, 5, 105, 0, 0, 1269, 1270, 5, 110, 0, 0, 1270, 1271, 5, 102, 0, 0, 1271, 1272, 5, 111, 0, 0, 1272, 339, 1, 0, 0, 0, 1273, 1274, 3, 58, 21, 0, 1274, 1275, 1, 0, 0, 0, 1275, 1276, 6, 162, 11, 0, 1276, 341, 1, 0, 0, 0, 1277, 1278, 3, 60, 22, 0, 1278, 1279, 1, 0, 0, 0, 1279, 1280, 6, 163, 11, 0, 1280, 343, 1, 0, 0, 0, 1281, 1282, 3, 62, 23, 0, 1282, 1283, 1, 0, 0, 0, 1283, 1284, 6, 164, 11, 0, 1284, 345, 1, 0, 0, 0, 1285, 1286, 3, 78, 31, 0, 1286, 1287, 1, 0, 0, 0, 1287, 1288, 6, 165, 14, 0, 1288, 1289, 6, 165, 15, 0, 1289, 347, 1, 0, 0, 0, 1290, 1291, 5, 102, 0, 0, 1291, 1292, 5, 117, 0, 0, 1292, 1293, 5, 110, 0, 0, 1293, 1294, 5, 99, 0, 0, 1294, 1295, 5, 116, 0, 0, 1295, 1296, 5, 105, 0, 0, 1296, 1297, 5, 111, 0, 0, 1297, 1298, 5, 110, 0, 0, 1298, 1299, 5, 115, 0, 0, 1299, 349, 1, 0, 0, 0, 1300, 1301, 3, 58, 21, 0, 1301, 1302, 1, 0, 0, 0, 1302, 1303, 6, 167, 11, 0, 1303, 351, 1, 0, 0, 0, 1304, 1305, 3, 60, 22, 0, 1305, 1306, 1, 0, 0, 0, 1306, 1307, 6, 168, 11, 0, 1307, 353, 1, 0, 0, 0, 1308, 1309, 3, 62, 23, 0, 1309, 1310, 1, 0, 0, 0, 1310, 1311, 6, 169, 11, 0, 1311, 355, 1, 0, 0, 0, 1312, 1313, 3, 178, 81, 0, 1313, 1314, 1, 0, 0, 0, 1314, 1315, 6, 170, 16, 0, 1315, 1316, 6, 170, 15, 0, 1316, 357, 1, 0, 0, 0, 1317, 1318, 5, 58, 0, 0, 1318, 359, 1, 0, 0, 0, 1319, 1325, 3, 90, 37, 0, 1320, 1325, 3, 80, 32, 0, 1321, 1325, 3, 120, 52, 0, 1322, 1325, 3, 82, 33, 0, 1323, 1325, 3, 96, 40, 0, 1324, 1319, 1, 0, 0, 0, 1324, 1320, 1, 0, 0, 0, 1324, 1321, 1, 0, 0, 0, 1324, 1322, 1, 0, 0, 0, 1324, 1323, 1, 0, 0, 0, 1325, 1326, 1, 0, 0, 0, 1326, 1324, 1, 0, 0, 0, 1326, 1327, 1, 0, 0, 0, 1327, 361, 1, 0, 0, 0, 1328, 1329, 3, 58, 21, 0, 1329, 1330, 1, 0, 0, 0, 1330, 1331, 6, 173, 11, 0, 1331, 363, 1, 0, 0, 0, 1332, 1333, 3, 60, 22, 0, 1333, 1334, 1, 0, 0, 0, 1334, 1335, 6, 174, 11, 0, 1335, 365, 1, 0, 0, 0, 1336, 1337, 3, 62, 23, 0, 1337, 1338, 1, 0, 0, 0, 1338, 1339, 6, 175, 11, 0, 1339, 367, 1, 0, 0, 0, 1340, 1341, 3, 78, 31, 0, 1341, 1342, 1, 0, 0, 0, 1342, 1343, 6, 176, 14, 0, 1343, 1344, 6, 176, 15, 0, 1344, 369, 1, 0, 0, 0, 1345, 1346, 3, 66, 25, 0, 1346, 1347, 1, 0, 0, 0, 1347, 1348, 6, 177, 20, 0, 1348, 1349, 6, 177, 15, 0, 1349, 1350, 6, 177, 31, 0, 1350, 371, 1, 0, 0, 0, 1351, 1352, 3, 58, 21, 0, 1352, 1353, 1, 0, 0, 0, 1353, 1354, 6, 178, 11, 0, 1354, 373, 1, 0, 0, 0, 1355, 1356, 3, 60, 22, 0, 1356, 1357, 1, 0, 0, 0, 1357, 1358, 6, 179, 11, 0, 1358, 375, 1, 0, 0, 0, 1359, 1360, 3, 62, 23, 0, 1360, 1361, 1, 0, 0, 0, 1361, 1362, 6, 180, 11, 0, 1362, 377, 1, 0, 0, 0, 1363, 1364, 3, 116, 50, 0, 1364, 1365, 1, 0, 0, 0, 1365, 1366, 6, 181, 17, 0, 1366, 1367, 6, 181, 15, 0, 1367, 1368, 6, 181, 7, 0, 1368, 379, 1, 0, 0, 0, 1369, 1370, 3, 58, 21, 0, 1370, 1371, 1, 0, 0, 0, 1371, 1372, 6, 182, 11, 0, 1372, 381, 1, 0, 0, 0, 1373, 1374, 3, 60, 22, 0, 1374, 1375, 1, 0, 0, 0, 1375, 1376, 6, 183, 11, 0, 1376, 383, 1, 0, 0, 0, 1377, 1378, 3, 62, 23, 0, 1378, 1379, 1, 0, 0, 0, 1379, 1380, 6, 184, 11, 0, 1380, 385, 1, 0, 0, 0, 1381, 1382, 3, 184, 84, 0, 1382, 1383, 1, 0, 0, 0, 1383, 1384, 6, 185, 15, 0, 1384, 1385, 6, 185, 0, 0, 1385, 1386, 6, 185, 25, 0, 1386, 387, 1, 0, 0, 0, 1387, 1388, 3, 180, 82, 0, 1388, 1389, 1, 0, 0, 0, 1389, 1390, 6, 186, 15, 0, 1390, 1391, 6, 186, 0, 0, 1391, 1392, 6, 186, 30, 0, 1392, 389, 1, 0, 0, 0, 1393, 1394, 3, 106, 45, 0, 1394, 1395, 1, 0, 0, 0, 1395, 1396, 6, 187, 15, 0, 1396, 1397, 6, 187, 0, 0, 1397, 1398, 6, 187, 32, 0, 1398, 391, 1, 0, 0, 0, 1399, 1400, 3, 78, 31, 0, 1400, 1401, 1, 0, 0, 0, 1401, 1402, 6, 188, 14, 0, 1402, 1403, 6, 188, 15, 0, 1403, 393, 1, 0, 0, 0, 62, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 566, 576, 580, 583, 592, 594, 605, 612, 617, 656, 661, 670, 677, 682, 684, 695, 703, 706, 708, 713, 718, 724, 731, 736, 742, 745, 753, 757, 890, 895, 900, 902, 908, 993, 997, 1002, 1007, 1012, 1014, 1018, 1020, 1097, 1101, 1106, 1324, 1326, 33, 5, 2, 0, 5, 4, 0, 5, 6, 0, 5, 1, 0, 5, 3, 0, 5, 8, 0, 5, 12, 0, 5, 14, 0, 5, 10, 0, 5, 5, 0, 5, 11, 0, 0, 1, 0, 7, 68, 0, 5, 0, 0, 7, 29, 0, 4, 0, 0, 7, 69, 0, 7, 38, 0, 7, 36, 0, 7, 30, 0, 7, 25, 0, 7, 40, 0, 7, 79, 0, 5, 13, 0, 5, 7, 0, 7, 71, 0, 7, 89, 0, 7, 88, 0, 7, 87, 0, 5, 9, 0, 7, 70, 0, 5, 15, 0, 7, 33, 0]
\ No newline at end of file
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
index 3f769b2614479..f9ca7ff95c847 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
@@ -18,48 +18,52 @@ public class EsqlBaseLexer extends Lexer {
     new PredictionContextCache();
   public static final int
     DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, INLINESTATS=8, 
-    KEEP=9, LIMIT=10, META=11, METRICS=12, MV_EXPAND=13, RENAME=14, ROW=15, 
-    SHOW=16, SORT=17, STATS=18, WHERE=19, UNKNOWN_CMD=20, LINE_COMMENT=21, 
-    MULTILINE_COMMENT=22, WS=23, INDEX_UNQUOTED_IDENTIFIER=24, EXPLAIN_WS=25, 
-    EXPLAIN_LINE_COMMENT=26, EXPLAIN_MULTILINE_COMMENT=27, PIPE=28, QUOTED_STRING=29, 
-    INTEGER_LITERAL=30, DECIMAL_LITERAL=31, BY=32, AND=33, ASC=34, ASSIGN=35, 
-    CAST_OP=36, COMMA=37, DESC=38, DOT=39, FALSE=40, FIRST=41, LAST=42, LP=43, 
-    IN=44, IS=45, LIKE=46, NOT=47, NULL=48, NULLS=49, OR=50, PARAM=51, RLIKE=52, 
-    RP=53, TRUE=54, EQ=55, CIEQ=56, NEQ=57, LT=58, LTE=59, GT=60, GTE=61, 
-    PLUS=62, MINUS=63, ASTERISK=64, SLASH=65, PERCENT=66, OPENING_BRACKET=67, 
-    CLOSING_BRACKET=68, UNQUOTED_IDENTIFIER=69, QUOTED_IDENTIFIER=70, EXPR_LINE_COMMENT=71, 
-    EXPR_MULTILINE_COMMENT=72, EXPR_WS=73, METADATA=74, FROM_LINE_COMMENT=75, 
-    FROM_MULTILINE_COMMENT=76, FROM_WS=77, ID_PATTERN=78, PROJECT_LINE_COMMENT=79, 
-    PROJECT_MULTILINE_COMMENT=80, PROJECT_WS=81, AS=82, RENAME_LINE_COMMENT=83, 
-    RENAME_MULTILINE_COMMENT=84, RENAME_WS=85, ON=86, WITH=87, ENRICH_POLICY_NAME=88, 
-    ENRICH_LINE_COMMENT=89, ENRICH_MULTILINE_COMMENT=90, ENRICH_WS=91, ENRICH_FIELD_LINE_COMMENT=92, 
-    ENRICH_FIELD_MULTILINE_COMMENT=93, ENRICH_FIELD_WS=94, MVEXPAND_LINE_COMMENT=95, 
-    MVEXPAND_MULTILINE_COMMENT=96, MVEXPAND_WS=97, INFO=98, SHOW_LINE_COMMENT=99, 
-    SHOW_MULTILINE_COMMENT=100, SHOW_WS=101, FUNCTIONS=102, META_LINE_COMMENT=103, 
-    META_MULTILINE_COMMENT=104, META_WS=105, COLON=106, SETTING=107, SETTING_LINE_COMMENT=108, 
-    SETTTING_MULTILINE_COMMENT=109, SETTING_WS=110, METRICS_LINE_COMMENT=111, 
-    METRICS_MULTILINE_COMMENT=112, METRICS_WS=113, CLOSING_METRICS_LINE_COMMENT=114, 
-    CLOSING_METRICS_MULTILINE_COMMENT=115, CLOSING_METRICS_WS=116;
+    KEEP=9, LIMIT=10, LOOKUP=11, META=12, METRICS=13, MV_EXPAND=14, RENAME=15, 
+    ROW=16, SHOW=17, SORT=18, STATS=19, WHERE=20, UNKNOWN_CMD=21, LINE_COMMENT=22, 
+    MULTILINE_COMMENT=23, WS=24, INDEX_UNQUOTED_IDENTIFIER=25, EXPLAIN_WS=26, 
+    EXPLAIN_LINE_COMMENT=27, EXPLAIN_MULTILINE_COMMENT=28, PIPE=29, QUOTED_STRING=30, 
+    INTEGER_LITERAL=31, DECIMAL_LITERAL=32, BY=33, AND=34, ASC=35, ASSIGN=36, 
+    CAST_OP=37, COMMA=38, DESC=39, DOT=40, FALSE=41, FIRST=42, LAST=43, LP=44, 
+    IN=45, IS=46, LIKE=47, NOT=48, NULL=49, NULLS=50, OR=51, PARAM=52, RLIKE=53, 
+    RP=54, TRUE=55, EQ=56, CIEQ=57, NEQ=58, LT=59, LTE=60, GT=61, GTE=62, 
+    PLUS=63, MINUS=64, ASTERISK=65, SLASH=66, PERCENT=67, OPENING_BRACKET=68, 
+    CLOSING_BRACKET=69, UNQUOTED_IDENTIFIER=70, QUOTED_IDENTIFIER=71, EXPR_LINE_COMMENT=72, 
+    EXPR_MULTILINE_COMMENT=73, EXPR_WS=74, METADATA=75, FROM_LINE_COMMENT=76, 
+    FROM_MULTILINE_COMMENT=77, FROM_WS=78, ID_PATTERN=79, PROJECT_LINE_COMMENT=80, 
+    PROJECT_MULTILINE_COMMENT=81, PROJECT_WS=82, AS=83, RENAME_LINE_COMMENT=84, 
+    RENAME_MULTILINE_COMMENT=85, RENAME_WS=86, ON=87, WITH=88, ENRICH_POLICY_NAME=89, 
+    ENRICH_LINE_COMMENT=90, ENRICH_MULTILINE_COMMENT=91, ENRICH_WS=92, ENRICH_FIELD_LINE_COMMENT=93, 
+    ENRICH_FIELD_MULTILINE_COMMENT=94, ENRICH_FIELD_WS=95, LOOKUP_LINE_COMMENT=96, 
+    LOOKUP_MULTILINE_COMMENT=97, LOOKUP_WS=98, LOOKUP_FIELD_LINE_COMMENT=99, 
+    LOOKUP_FIELD_MULTILINE_COMMENT=100, LOOKUP_FIELD_WS=101, MVEXPAND_LINE_COMMENT=102, 
+    MVEXPAND_MULTILINE_COMMENT=103, MVEXPAND_WS=104, INFO=105, SHOW_LINE_COMMENT=106, 
+    SHOW_MULTILINE_COMMENT=107, SHOW_WS=108, FUNCTIONS=109, META_LINE_COMMENT=110, 
+    META_MULTILINE_COMMENT=111, META_WS=112, COLON=113, SETTING=114, SETTING_LINE_COMMENT=115, 
+    SETTTING_MULTILINE_COMMENT=116, SETTING_WS=117, METRICS_LINE_COMMENT=118, 
+    METRICS_MULTILINE_COMMENT=119, METRICS_WS=120, CLOSING_METRICS_LINE_COMMENT=121, 
+    CLOSING_METRICS_MULTILINE_COMMENT=122, CLOSING_METRICS_WS=123;
   public static final int
     EXPLAIN_MODE=1, EXPRESSION_MODE=2, FROM_MODE=3, PROJECT_MODE=4, RENAME_MODE=5, 
-    ENRICH_MODE=6, ENRICH_FIELD_MODE=7, MVEXPAND_MODE=8, SHOW_MODE=9, META_MODE=10, 
-    SETTING_MODE=11, METRICS_MODE=12, CLOSING_METRICS_MODE=13;
+    ENRICH_MODE=6, ENRICH_FIELD_MODE=7, LOOKUP_MODE=8, LOOKUP_FIELD_MODE=9, 
+    MVEXPAND_MODE=10, SHOW_MODE=11, META_MODE=12, SETTING_MODE=13, METRICS_MODE=14, 
+    CLOSING_METRICS_MODE=15;
   public static String[] channelNames = {
     "DEFAULT_TOKEN_CHANNEL", "HIDDEN"
   };
 
   public static String[] modeNames = {
     "DEFAULT_MODE", "EXPLAIN_MODE", "EXPRESSION_MODE", "FROM_MODE", "PROJECT_MODE", 
-    "RENAME_MODE", "ENRICH_MODE", "ENRICH_FIELD_MODE", "MVEXPAND_MODE", "SHOW_MODE", 
-    "META_MODE", "SETTING_MODE", "METRICS_MODE", "CLOSING_METRICS_MODE"
+    "RENAME_MODE", "ENRICH_MODE", "ENRICH_FIELD_MODE", "LOOKUP_MODE", "LOOKUP_FIELD_MODE", 
+    "MVEXPAND_MODE", "SHOW_MODE", "META_MODE", "SETTING_MODE", "METRICS_MODE", 
+    "CLOSING_METRICS_MODE"
   };
 
   private static String[] makeRuleNames() {
     return new String[] {
       "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "INLINESTATS", 
-      "KEEP", "LIMIT", "META", "METRICS", "MV_EXPAND", "RENAME", "ROW", "SHOW", 
-      "SORT", "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", 
-      "WS", "INDEX_UNQUOTED_IDENTIFIER_PART", "INDEX_UNQUOTED_IDENTIFIER", 
+      "KEEP", "LIMIT", "LOOKUP", "META", "METRICS", "MV_EXPAND", "RENAME", 
+      "ROW", "SHOW", "SORT", "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", 
+      "MULTILINE_COMMENT", "WS", "INDEX_UNQUOTED_IDENTIFIER_PART", "INDEX_UNQUOTED_IDENTIFIER", 
       "EXPLAIN_OPENING_BRACKET", "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", 
       "EXPLAIN_MULTILINE_COMMENT", "PIPE", "DIGIT", "LETTER", "ESCAPE_SEQUENCE", 
       "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", "BACKQUOTE_BLOCK", 
@@ -82,7 +86,11 @@ private static String[] makeRuleNames() {
       "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_PIPE", "ENRICH_FIELD_ASSIGN", 
       "ENRICH_FIELD_COMMA", "ENRICH_FIELD_DOT", "ENRICH_FIELD_WITH", "ENRICH_FIELD_ID_PATTERN", 
       "ENRICH_FIELD_QUOTED_IDENTIFIER", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", 
-      "ENRICH_FIELD_WS", "MVEXPAND_PIPE", "MVEXPAND_DOT", "MVEXPAND_QUOTED_IDENTIFIER", 
+      "ENRICH_FIELD_WS", "LOOKUP_PIPE", "LOOKUP_COMMA", "LOOKUP_DOT", "LOOKUP_ON", 
+      "LOOKUP_INDEX_UNQUOTED_IDENTIFIER", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", 
+      "LOOKUP_WS", "LOOKUP_FIELD_PIPE", "LOOKUP_FIELD_COMMA", "LOOKUP_FIELD_DOT", 
+      "LOOKUP_FIELD_ID_PATTERN", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", 
+      "LOOKUP_FIELD_WS", "MVEXPAND_PIPE", "MVEXPAND_DOT", "MVEXPAND_QUOTED_IDENTIFIER", 
       "MVEXPAND_UNQUOTED_IDENTIFIER", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", 
       "MVEXPAND_WS", "SHOW_PIPE", "INFO", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", 
       "SHOW_WS", "META_PIPE", "FUNCTIONS", "META_LINE_COMMENT", "META_MULTILINE_COMMENT", 
@@ -99,25 +107,26 @@ private static String[] makeRuleNames() {
   private static String[] makeLiteralNames() {
     return new String[] {
       null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", 
-      "'grok'", "'inlinestats'", "'keep'", "'limit'", "'meta'", "'metrics'", 
-      "'mv_expand'", "'rename'", "'row'", "'show'", "'sort'", "'stats'", "'where'", 
-      null, null, null, null, null, null, null, null, "'|'", null, null, null, 
-      "'by'", "'and'", "'asc'", "'='", "'::'", "','", "'desc'", "'.'", "'false'", 
-      "'first'", "'last'", "'('", "'in'", "'is'", "'like'", "'not'", "'null'", 
-      "'nulls'", "'or'", "'?'", "'rlike'", "')'", "'true'", "'=='", "'=~'", 
-      "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", 
-      null, "']'", null, null, null, null, null, "'metadata'", null, null, 
-      null, null, null, null, null, "'as'", null, null, null, "'on'", "'with'", 
-      null, null, null, null, null, null, null, null, null, null, "'info'", 
-      null, null, null, "'functions'", null, null, null, "':'"
+      "'grok'", "'inlinestats'", "'keep'", "'limit'", "'lookup'", "'meta'", 
+      "'metrics'", "'mv_expand'", "'rename'", "'row'", "'show'", "'sort'", 
+      "'stats'", "'where'", null, null, null, null, null, null, null, null, 
+      "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "','", 
+      "'desc'", "'.'", "'false'", "'first'", "'last'", "'('", "'in'", "'is'", 
+      "'like'", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", 
+      "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", 
+      "'-'", "'*'", "'/'", "'%'", null, "']'", null, null, null, null, null, 
+      "'metadata'", null, null, null, null, null, null, null, "'as'", null, 
+      null, null, "'on'", "'with'", null, null, null, null, null, null, null, 
+      null, null, null, null, null, null, null, null, null, "'info'", null, 
+      null, null, "'functions'", null, null, null, "':'"
     };
   }
   private static final String[] _LITERAL_NAMES = makeLiteralNames();
   private static String[] makeSymbolicNames() {
     return new String[] {
       null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", 
-      "INLINESTATS", "KEEP", "LIMIT", "META", "METRICS", "MV_EXPAND", "RENAME", 
-      "ROW", "SHOW", "SORT", "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", 
+      "INLINESTATS", "KEEP", "LIMIT", "LOOKUP", "META", "METRICS", "MV_EXPAND", 
+      "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", 
       "MULTILINE_COMMENT", "WS", "INDEX_UNQUOTED_IDENTIFIER", "EXPLAIN_WS", 
       "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", "PIPE", "QUOTED_STRING", 
       "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", 
@@ -131,7 +140,9 @@ private static String[] makeSymbolicNames() {
       "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", "RENAME_WS", 
       "ON", "WITH", "ENRICH_POLICY_NAME", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", 
       "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", 
-      "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", 
+      "ENRICH_FIELD_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", 
+      "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", 
+      "LOOKUP_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", 
       "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", 
       "SHOW_WS", "FUNCTIONS", "META_LINE_COMMENT", "META_MULTILINE_COMMENT", 
       "META_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", "SETTTING_MULTILINE_COMMENT", 
@@ -200,223 +211,243 @@ public EsqlBaseLexer(CharStream input) {
   public ATN getATN() { return _ATN; }
 
   public static final String _serializedATN =
-    "\u0004\u0000t\u0511\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
+    "\u0004\u0000{\u057c\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
+    "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
     "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
     "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+
-    "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0002\u0000\u0007"+
-    "\u0000\u0002\u0001\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007"+
-    "\u0003\u0002\u0004\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006\u0007"+
-    "\u0006\u0002\u0007\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n"+
-    "\u0007\n\u0002\u000b\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002"+
-    "\u000e\u0007\u000e\u0002\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002"+
-    "\u0011\u0007\u0011\u0002\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002"+
-    "\u0014\u0007\u0014\u0002\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002"+
-    "\u0017\u0007\u0017\u0002\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002"+
-    "\u001a\u0007\u001a\u0002\u001b\u0007\u001b\u0002\u001c\u0007\u001c\u0002"+
-    "\u001d\u0007\u001d\u0002\u001e\u0007\u001e\u0002\u001f\u0007\u001f\u0002"+
-    " \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002#\u0007#\u0002$\u0007$\u0002"+
-    "%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002(\u0007(\u0002)\u0007)\u0002"+
-    "*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002-\u0007-\u0002.\u0007.\u0002"+
-    "/\u0007/\u00020\u00070\u00021\u00071\u00022\u00072\u00023\u00073\u0002"+
-    "4\u00074\u00025\u00075\u00026\u00076\u00027\u00077\u00028\u00078\u0002"+
-    "9\u00079\u0002:\u0007:\u0002;\u0007;\u0002<\u0007<\u0002=\u0007=\u0002"+
-    ">\u0007>\u0002?\u0007?\u0002@\u0007@\u0002A\u0007A\u0002B\u0007B\u0002"+
-    "C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002F\u0007F\u0002G\u0007G\u0002"+
-    "H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002K\u0007K\u0002L\u0007L\u0002"+
-    "M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002P\u0007P\u0002Q\u0007Q\u0002"+
-    "R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002U\u0007U\u0002V\u0007V\u0002"+
-    "W\u0007W\u0002X\u0007X\u0002Y\u0007Y\u0002Z\u0007Z\u0002[\u0007[\u0002"+
-    "\\\u0007\\\u0002]\u0007]\u0002^\u0007^\u0002_\u0007_\u0002`\u0007`\u0002"+
-    "a\u0007a\u0002b\u0007b\u0002c\u0007c\u0002d\u0007d\u0002e\u0007e\u0002"+
-    "f\u0007f\u0002g\u0007g\u0002h\u0007h\u0002i\u0007i\u0002j\u0007j\u0002"+
-    "k\u0007k\u0002l\u0007l\u0002m\u0007m\u0002n\u0007n\u0002o\u0007o\u0002"+
-    "p\u0007p\u0002q\u0007q\u0002r\u0007r\u0002s\u0007s\u0002t\u0007t\u0002"+
-    "u\u0007u\u0002v\u0007v\u0002w\u0007w\u0002x\u0007x\u0002y\u0007y\u0002"+
-    "z\u0007z\u0002{\u0007{\u0002|\u0007|\u0002}\u0007}\u0002~\u0007~\u0002"+
-    "\u007f\u0007\u007f\u0002\u0080\u0007\u0080\u0002\u0081\u0007\u0081\u0002"+
-    "\u0082\u0007\u0082\u0002\u0083\u0007\u0083\u0002\u0084\u0007\u0084\u0002"+
-    "\u0085\u0007\u0085\u0002\u0086\u0007\u0086\u0002\u0087\u0007\u0087\u0002"+
-    "\u0088\u0007\u0088\u0002\u0089\u0007\u0089\u0002\u008a\u0007\u008a\u0002"+
-    "\u008b\u0007\u008b\u0002\u008c\u0007\u008c\u0002\u008d\u0007\u008d\u0002"+
-    "\u008e\u0007\u008e\u0002\u008f\u0007\u008f\u0002\u0090\u0007\u0090\u0002"+
-    "\u0091\u0007\u0091\u0002\u0092\u0007\u0092\u0002\u0093\u0007\u0093\u0002"+
-    "\u0094\u0007\u0094\u0002\u0095\u0007\u0095\u0002\u0096\u0007\u0096\u0002"+
-    "\u0097\u0007\u0097\u0002\u0098\u0007\u0098\u0002\u0099\u0007\u0099\u0002"+
-    "\u009a\u0007\u009a\u0002\u009b\u0007\u009b\u0002\u009c\u0007\u009c\u0002"+
-    "\u009d\u0007\u009d\u0002\u009e\u0007\u009e\u0002\u009f\u0007\u009f\u0002"+
-    "\u00a0\u0007\u00a0\u0002\u00a1\u0007\u00a1\u0002\u00a2\u0007\u00a2\u0002"+
-    "\u00a3\u0007\u00a3\u0002\u00a4\u0007\u00a4\u0002\u00a5\u0007\u00a5\u0002"+
-    "\u00a6\u0007\u00a6\u0002\u00a7\u0007\u00a7\u0002\u00a8\u0007\u00a8\u0002"+
-    "\u00a9\u0007\u00a9\u0002\u00aa\u0007\u00aa\u0002\u00ab\u0007\u00ab\u0002"+
-    "\u00ac\u0007\u00ac\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+
-    "\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+
-    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+
-    "\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+
-    "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001"+
-    "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0004\u0001"+
-    "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+
-    "\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+
-    "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006\u0001"+
-    "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007\u0001"+
-    "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+
-    "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+
-    "\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\t"+
-    "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001"+
-    "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001"+
-    "\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+
-    "\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+
-    "\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001"+
-    "\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001"+
-    "\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001"+
-    "\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001"+
-    "\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001"+
-    "\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001"+
-    "\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001"+
-    "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0004\u0013\u020a"+
-    "\b\u0013\u000b\u0013\f\u0013\u020b\u0001\u0013\u0001\u0013\u0001\u0014"+
-    "\u0001\u0014\u0001\u0014\u0001\u0014\u0005\u0014\u0214\b\u0014\n\u0014"+
-    "\f\u0014\u0217\t\u0014\u0001\u0014\u0003\u0014\u021a\b\u0014\u0001\u0014"+
-    "\u0003\u0014\u021d\b\u0014\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015"+
-    "\u0001\u0015\u0001\u0015\u0001\u0015\u0005\u0015\u0226\b\u0015\n\u0015"+
-    "\f\u0015\u0229\t\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015"+
-    "\u0001\u0015\u0001\u0016\u0004\u0016\u0231\b\u0016\u000b\u0016\f\u0016"+
-    "\u0232\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0003"+
-    "\u0017\u023a\b\u0017\u0001\u0018\u0004\u0018\u023d\b\u0018\u000b\u0018"+
-    "\f\u0018\u023e\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019"+
-    "\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b"+
-    "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001c"+
-    "\u0001\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001e"+
-    "\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001 \u0001"+
-    " \u0001!\u0001!\u0001!\u0001\"\u0001\"\u0001#\u0001#\u0003#\u0266\b#\u0001"+
-    "#\u0004#\u0269\b#\u000b#\f#\u026a\u0001$\u0001$\u0001%\u0001%\u0001&\u0001"+
-    "&\u0001&\u0003&\u0274\b&\u0001\'\u0001\'\u0001(\u0001(\u0001(\u0003(\u027b"+
-    "\b(\u0001)\u0001)\u0001)\u0005)\u0280\b)\n)\f)\u0283\t)\u0001)\u0001)"+
-    "\u0001)\u0001)\u0001)\u0001)\u0005)\u028b\b)\n)\f)\u028e\t)\u0001)\u0001"+
-    ")\u0001)\u0001)\u0001)\u0003)\u0295\b)\u0001)\u0003)\u0298\b)\u0003)\u029a"+
-    "\b)\u0001*\u0004*\u029d\b*\u000b*\f*\u029e\u0001+\u0004+\u02a2\b+\u000b"+
-    "+\f+\u02a3\u0001+\u0001+\u0005+\u02a8\b+\n+\f+\u02ab\t+\u0001+\u0001+"+
-    "\u0004+\u02af\b+\u000b+\f+\u02b0\u0001+\u0004+\u02b4\b+\u000b+\f+\u02b5"+
-    "\u0001+\u0001+\u0005+\u02ba\b+\n+\f+\u02bd\t+\u0003+\u02bf\b+\u0001+\u0001"+
-    "+\u0001+\u0001+\u0004+\u02c5\b+\u000b+\f+\u02c6\u0001+\u0001+\u0003+\u02cb"+
-    "\b+\u0001,\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001"+
-    ".\u0001.\u0001/\u0001/\u00010\u00010\u00010\u00011\u00011\u00012\u0001"+
-    "2\u00012\u00012\u00012\u00013\u00013\u00014\u00014\u00014\u00014\u0001"+
-    "4\u00014\u00015\u00015\u00015\u00015\u00015\u00015\u00016\u00016\u0001"+
-    "6\u00016\u00016\u00017\u00017\u00018\u00018\u00018\u00019\u00019\u0001"+
-    "9\u0001:\u0001:\u0001:\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001"+
-    "<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001=\u0001=\u0001"+
-    "=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001@\u0001@\u0001@\u0001@\u0001"+
-    "@\u0001@\u0001A\u0001A\u0001B\u0001B\u0001B\u0001B\u0001B\u0001C\u0001"+
-    "C\u0001C\u0001D\u0001D\u0001D\u0001E\u0001E\u0001E\u0001F\u0001F\u0001"+
-    "G\u0001G\u0001G\u0001H\u0001H\u0001I\u0001I\u0001I\u0001J\u0001J\u0001"+
-    "K\u0001K\u0001L\u0001L\u0001M\u0001M\u0001N\u0001N\u0001O\u0001O\u0001"+
-    "O\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0005"+
-    "Q\u034e\bQ\nQ\fQ\u0351\tQ\u0001Q\u0001Q\u0003Q\u0355\bQ\u0001Q\u0004Q"+
-    "\u0358\bQ\u000bQ\fQ\u0359\u0003Q\u035c\bQ\u0001R\u0001R\u0004R\u0360\b"+
-    "R\u000bR\fR\u0361\u0001R\u0001R\u0001S\u0001S\u0001T\u0001T\u0001T\u0001"+
-    "T\u0001U\u0001U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001W\u0001"+
-    "W\u0001W\u0001W\u0001W\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001Y\u0001"+
-    "Y\u0001Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001"+
-    "\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001]\u0001]\u0001"+
-    "]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001"+
-    "_\u0001`\u0001`\u0001`\u0001`\u0001a\u0001a\u0001a\u0001a\u0001b\u0001"+
-    "b\u0001b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001"+
-    "d\u0001d\u0001e\u0001e\u0001e\u0001e\u0003e\u03b7\be\u0001f\u0001f\u0003"+
-    "f\u03bb\bf\u0001f\u0005f\u03be\bf\nf\ff\u03c1\tf\u0001f\u0001f\u0003f"+
-    "\u03c5\bf\u0001f\u0004f\u03c8\bf\u000bf\ff\u03c9\u0003f\u03cc\bf\u0001"+
-    "g\u0001g\u0004g\u03d0\bg\u000bg\fg\u03d1\u0001h\u0001h\u0001h\u0001h\u0001"+
-    "i\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001k\u0001k\u0001"+
-    "k\u0001k\u0001k\u0001l\u0001l\u0001l\u0001l\u0001m\u0001m\u0001m\u0001"+
-    "m\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001p\u0001p\u0001"+
+    "\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+
+    "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+
+    "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+
+    "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+
+    "\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0002\u000f\u0007\u000f"+
+    "\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011\u0002\u0012\u0007\u0012"+
+    "\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014\u0002\u0015\u0007\u0015"+
+    "\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017\u0002\u0018\u0007\u0018"+
+    "\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a\u0002\u001b\u0007\u001b"+
+    "\u0002\u001c\u0007\u001c\u0002\u001d\u0007\u001d\u0002\u001e\u0007\u001e"+
+    "\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002"+
+    "#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002"+
+    "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+
+    "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+
+    "2\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u0002"+
+    "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007;\u0002"+
+    "<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0002@\u0007@\u0002"+
+    "A\u0007A\u0002B\u0007B\u0002C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002"+
+    "F\u0007F\u0002G\u0007G\u0002H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002"+
+    "K\u0007K\u0002L\u0007L\u0002M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002"+
+    "P\u0007P\u0002Q\u0007Q\u0002R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002"+
+    "U\u0007U\u0002V\u0007V\u0002W\u0007W\u0002X\u0007X\u0002Y\u0007Y\u0002"+
+    "Z\u0007Z\u0002[\u0007[\u0002\\\u0007\\\u0002]\u0007]\u0002^\u0007^\u0002"+
+    "_\u0007_\u0002`\u0007`\u0002a\u0007a\u0002b\u0007b\u0002c\u0007c\u0002"+
+    "d\u0007d\u0002e\u0007e\u0002f\u0007f\u0002g\u0007g\u0002h\u0007h\u0002"+
+    "i\u0007i\u0002j\u0007j\u0002k\u0007k\u0002l\u0007l\u0002m\u0007m\u0002"+
+    "n\u0007n\u0002o\u0007o\u0002p\u0007p\u0002q\u0007q\u0002r\u0007r\u0002"+
+    "s\u0007s\u0002t\u0007t\u0002u\u0007u\u0002v\u0007v\u0002w\u0007w\u0002"+
+    "x\u0007x\u0002y\u0007y\u0002z\u0007z\u0002{\u0007{\u0002|\u0007|\u0002"+
+    "}\u0007}\u0002~\u0007~\u0002\u007f\u0007\u007f\u0002\u0080\u0007\u0080"+
+    "\u0002\u0081\u0007\u0081\u0002\u0082\u0007\u0082\u0002\u0083\u0007\u0083"+
+    "\u0002\u0084\u0007\u0084\u0002\u0085\u0007\u0085\u0002\u0086\u0007\u0086"+
+    "\u0002\u0087\u0007\u0087\u0002\u0088\u0007\u0088\u0002\u0089\u0007\u0089"+
+    "\u0002\u008a\u0007\u008a\u0002\u008b\u0007\u008b\u0002\u008c\u0007\u008c"+
+    "\u0002\u008d\u0007\u008d\u0002\u008e\u0007\u008e\u0002\u008f\u0007\u008f"+
+    "\u0002\u0090\u0007\u0090\u0002\u0091\u0007\u0091\u0002\u0092\u0007\u0092"+
+    "\u0002\u0093\u0007\u0093\u0002\u0094\u0007\u0094\u0002\u0095\u0007\u0095"+
+    "\u0002\u0096\u0007\u0096\u0002\u0097\u0007\u0097\u0002\u0098\u0007\u0098"+
+    "\u0002\u0099\u0007\u0099\u0002\u009a\u0007\u009a\u0002\u009b\u0007\u009b"+
+    "\u0002\u009c\u0007\u009c\u0002\u009d\u0007\u009d\u0002\u009e\u0007\u009e"+
+    "\u0002\u009f\u0007\u009f\u0002\u00a0\u0007\u00a0\u0002\u00a1\u0007\u00a1"+
+    "\u0002\u00a2\u0007\u00a2\u0002\u00a3\u0007\u00a3\u0002\u00a4\u0007\u00a4"+
+    "\u0002\u00a5\u0007\u00a5\u0002\u00a6\u0007\u00a6\u0002\u00a7\u0007\u00a7"+
+    "\u0002\u00a8\u0007\u00a8\u0002\u00a9\u0007\u00a9\u0002\u00aa\u0007\u00aa"+
+    "\u0002\u00ab\u0007\u00ab\u0002\u00ac\u0007\u00ac\u0002\u00ad\u0007\u00ad"+
+    "\u0002\u00ae\u0007\u00ae\u0002\u00af\u0007\u00af\u0002\u00b0\u0007\u00b0"+
+    "\u0002\u00b1\u0007\u00b1\u0002\u00b2\u0007\u00b2\u0002\u00b3\u0007\u00b3"+
+    "\u0002\u00b4\u0007\u00b4\u0002\u00b5\u0007\u00b5\u0002\u00b6\u0007\u00b6"+
+    "\u0002\u00b7\u0007\u00b7\u0002\u00b8\u0007\u00b8\u0002\u00b9\u0007\u00b9"+
+    "\u0002\u00ba\u0007\u00ba\u0002\u00bb\u0007\u00bb\u0002\u00bc\u0007\u00bc"+
+    "\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000"+
+    "\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001"+
+    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002"+
+    "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"+
+    "\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+
+    "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0004\u0001\u0004\u0001\u0004"+
+    "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"+
+    "\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+
+    "\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+
+    "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007\u0001\u0007\u0001\u0007"+
+    "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007"+
+    "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001"+
+    "\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001"+
+    "\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+
+    "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b"+
+    "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001"+
+    "\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\r\u0001"+
+    "\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001"+
+    "\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e"+
+    "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+
+    "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010"+
+    "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011"+
+    "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+
+    "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+
+    "\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+
+    "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0004\u0014"+
+    "\u0235\b\u0014\u000b\u0014\f\u0014\u0236\u0001\u0014\u0001\u0014\u0001"+
+    "\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0005\u0015\u023f\b\u0015\n"+
+    "\u0015\f\u0015\u0242\t\u0015\u0001\u0015\u0003\u0015\u0245\b\u0015\u0001"+
+    "\u0015\u0003\u0015\u0248\b\u0015\u0001\u0015\u0001\u0015\u0001\u0016\u0001"+
+    "\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0005\u0016\u0251\b\u0016\n"+
+    "\u0016\f\u0016\u0254\t\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001"+
+    "\u0016\u0001\u0016\u0001\u0017\u0004\u0017\u025c\b\u0017\u000b\u0017\f"+
+    "\u0017\u025d\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018"+
+    "\u0003\u0018\u0265\b\u0018\u0001\u0019\u0004\u0019\u0268\b\u0019\u000b"+
+    "\u0019\f\u0019\u0269\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001"+
+    "\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001"+
+    "\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d\u0001"+
+    "\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001"+
+    "\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001 \u0001 \u0001!\u0001"+
+    "!\u0001\"\u0001\"\u0001\"\u0001#\u0001#\u0001$\u0001$\u0003$\u0291\b$"+
+    "\u0001$\u0004$\u0294\b$\u000b$\f$\u0295\u0001%\u0001%\u0001&\u0001&\u0001"+
+    "\'\u0001\'\u0001\'\u0003\'\u029f\b\'\u0001(\u0001(\u0001)\u0001)\u0001"+
+    ")\u0003)\u02a6\b)\u0001*\u0001*\u0001*\u0005*\u02ab\b*\n*\f*\u02ae\t*"+
+    "\u0001*\u0001*\u0001*\u0001*\u0001*\u0001*\u0005*\u02b6\b*\n*\f*\u02b9"+
+    "\t*\u0001*\u0001*\u0001*\u0001*\u0001*\u0003*\u02c0\b*\u0001*\u0003*\u02c3"+
+    "\b*\u0003*\u02c5\b*\u0001+\u0004+\u02c8\b+\u000b+\f+\u02c9\u0001,\u0004"+
+    ",\u02cd\b,\u000b,\f,\u02ce\u0001,\u0001,\u0005,\u02d3\b,\n,\f,\u02d6\t"+
+    ",\u0001,\u0001,\u0004,\u02da\b,\u000b,\f,\u02db\u0001,\u0004,\u02df\b"+
+    ",\u000b,\f,\u02e0\u0001,\u0001,\u0005,\u02e5\b,\n,\f,\u02e8\t,\u0003,"+
+    "\u02ea\b,\u0001,\u0001,\u0001,\u0001,\u0004,\u02f0\b,\u000b,\f,\u02f1"+
+    "\u0001,\u0001,\u0003,\u02f6\b,\u0001-\u0001-\u0001-\u0001.\u0001.\u0001"+
+    ".\u0001.\u0001/\u0001/\u0001/\u0001/\u00010\u00010\u00011\u00011\u0001"+
+    "1\u00012\u00012\u00013\u00013\u00013\u00013\u00013\u00014\u00014\u0001"+
+    "5\u00015\u00015\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u0001"+
+    "6\u00016\u00017\u00017\u00017\u00017\u00017\u00018\u00018\u00019\u0001"+
+    "9\u00019\u0001:\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001;\u0001"+
+    "<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001=\u0001=\u0001>\u0001"+
+    ">\u0001>\u0001>\u0001>\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001"+
+    "A\u0001A\u0001A\u0001A\u0001A\u0001A\u0001B\u0001B\u0001C\u0001C\u0001"+
+    "C\u0001C\u0001C\u0001D\u0001D\u0001D\u0001E\u0001E\u0001E\u0001F\u0001"+
+    "F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001H\u0001I\u0001I\u0001J\u0001"+
+    "J\u0001J\u0001K\u0001K\u0001L\u0001L\u0001M\u0001M\u0001N\u0001N\u0001"+
+    "O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0001Q\u0001"+
+    "Q\u0001Q\u0001R\u0001R\u0005R\u0379\bR\nR\fR\u037c\tR\u0001R\u0001R\u0003"+
+    "R\u0380\bR\u0001R\u0004R\u0383\bR\u000bR\fR\u0384\u0003R\u0387\bR\u0001"+
+    "S\u0001S\u0004S\u038b\bS\u000bS\fS\u038c\u0001S\u0001S\u0001T\u0001T\u0001"+
+    "U\u0001U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001"+
+    "W\u0001W\u0001X\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001"+
+    "Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001"+
+    "\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001"+
+    "^\u0001^\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001_\u0001"+
+    "`\u0001`\u0001`\u0001`\u0001a\u0001a\u0001a\u0001a\u0001b\u0001b\u0001"+
+    "b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001"+
+    "d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0003f\u03e2"+
+    "\bf\u0001g\u0001g\u0003g\u03e6\bg\u0001g\u0005g\u03e9\bg\ng\fg\u03ec\t"+
+    "g\u0001g\u0001g\u0003g\u03f0\bg\u0001g\u0004g\u03f3\bg\u000bg\fg\u03f4"+
+    "\u0003g\u03f7\bg\u0001h\u0001h\u0004h\u03fb\bh\u000bh\fh\u03fc\u0001i"+
+    "\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001j\u0001k\u0001k\u0001"+
+    "k\u0001k\u0001l\u0001l\u0001l\u0001l\u0001l\u0001m\u0001m\u0001m\u0001"+
+    "m\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001p\u0001"+
     "p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001r\u0001r\u0001"+
-    "s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001t\u0001t\u0001u\u0001"+
-    "u\u0001u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001v\u0001w\u0001"+
-    "w\u0001w\u0001w\u0001w\u0001w\u0001w\u0001x\u0001x\u0001y\u0004y\u041d"+
-    "\by\u000by\fy\u041e\u0001y\u0001y\u0003y\u0423\by\u0001y\u0004y\u0426"+
-    "\by\u000by\fy\u0427\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001"+
+    "s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001t\u0001u\u0001u\u0001"+
+    "u\u0001u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001v\u0001w\u0001w\u0001"+
+    "w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001x\u0001x\u0001x\u0001"+
+    "y\u0001y\u0001z\u0004z\u0448\bz\u000bz\fz\u0449\u0001z\u0001z\u0003z\u044e"+
+    "\bz\u0001z\u0004z\u0451\bz\u000bz\fz\u0452\u0001{\u0001{\u0001{\u0001"+
     "{\u0001|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001~\u0001"+
     "~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001"+
-    "\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0080\u0001"+
+    "\u0080\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0080\u0001"+
     "\u0081\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0082\u0001\u0082\u0001"+
     "\u0082\u0001\u0082\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001"+
     "\u0084\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001"+
     "\u0085\u0001\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001"+
     "\u0087\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001\u0088\u0001"+
     "\u0088\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001"+
-    "\u0089\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001"+
+    "\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001"+
     "\u008b\u0001\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001"+
-    "\u008c\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001"+
-    "\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001"+
-    "\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001"+
-    "\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001"+
-    "\u0092\u0001\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001\u0093\u0001"+
-    "\u0093\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001"+
+    "\u008c\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001"+
+    "\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001"+
+    "\u008f\u0001\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001"+
+    "\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092\u0001"+
+    "\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001"+
+    "\u0093\u0001\u0093\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001"+
     "\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0096\u0001\u0096\u0001"+
-    "\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096\u0001"+
     "\u0096\u0001\u0096\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0097\u0001"+
     "\u0098\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0099\u0001\u0099\u0001"+
-    "\u0099\u0001\u0099\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001"+
-    "\u009a\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c\u0001"+
-    "\u009c\u0001\u009c\u0004\u009c\u04c2\b\u009c\u000b\u009c\f\u009c\u04c3"+
-    "\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009e\u0001\u009e"+
-    "\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u009f"+
-    "\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a1"+
-    "\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a2"+
-    "\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a3\u0001\u00a3\u0001\u00a3"+
-    "\u0001\u00a3\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a5"+
-    "\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a6"+
-    "\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7\u0001\u00a7\u0001\u00a7"+
-    "\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a9"+
-    "\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00aa"+
-    "\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab"+
-    "\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ac"+
-    "\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0002\u0227\u028c\u0000"+
-    "\u00ad\u000e\u0001\u0010\u0002\u0012\u0003\u0014\u0004\u0016\u0005\u0018"+
-    "\u0006\u001a\u0007\u001c\b\u001e\t \n\"\u000b$\f&\r(\u000e*\u000f,\u0010"+
-    ".\u00110\u00122\u00134\u00146\u00158\u0016:\u0017<\u0000>\u0018@\u0000"+
-    "B\u0000D\u0019F\u001aH\u001bJ\u001cL\u0000N\u0000P\u0000R\u0000T\u0000"+
-    "V\u0000X\u0000Z\u0000\\\u0000^\u0000`\u001db\u001ed\u001ff h!j\"l#n$p"+
-    "%r&t\'v(x)z*|+~,\u0080-\u0082.\u0084/\u00860\u00881\u008a2\u008c3\u008e"+
-    "4\u00905\u00926\u00947\u00968\u00989\u009a:\u009c;\u009e<\u00a0=\u00a2"+
-    ">\u00a4?\u00a6@\u00a8A\u00aaB\u00acC\u00aeD\u00b0E\u00b2\u0000\u00b4F"+
-    "\u00b6G\u00b8H\u00baI\u00bc\u0000\u00be\u0000\u00c0\u0000\u00c2\u0000"+
-    "\u00c4\u0000\u00c6\u0000\u00c8J\u00ca\u0000\u00ccK\u00ceL\u00d0M\u00d2"+
-    "\u0000\u00d4\u0000\u00d6\u0000\u00d8\u0000\u00da\u0000\u00dcN\u00deO\u00e0"+
-    "P\u00e2Q\u00e4\u0000\u00e6\u0000\u00e8\u0000\u00ea\u0000\u00ecR\u00ee"+
-    "\u0000\u00f0S\u00f2T\u00f4U\u00f6\u0000\u00f8\u0000\u00faV\u00fcW\u00fe"+
-    "\u0000\u0100X\u0102\u0000\u0104\u0000\u0106Y\u0108Z\u010a[\u010c\u0000"+
-    "\u010e\u0000\u0110\u0000\u0112\u0000\u0114\u0000\u0116\u0000\u0118\u0000"+
-    "\u011a\\\u011c]\u011e^\u0120\u0000\u0122\u0000\u0124\u0000\u0126\u0000"+
-    "\u0128_\u012a`\u012ca\u012e\u0000\u0130b\u0132c\u0134d\u0136e\u0138\u0000"+
-    "\u013af\u013cg\u013eh\u0140i\u0142\u0000\u0144j\u0146k\u0148l\u014am\u014c"+
-    "n\u014e\u0000\u0150\u0000\u0152o\u0154p\u0156q\u0158\u0000\u015ar\u015c"+
-    "s\u015et\u0160\u0000\u0162\u0000\u0164\u0000\u0166\u0000\u000e\u0000\u0001"+
-    "\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\r\u0006\u0000\t\n"+
-    "\r\r  //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r\r  \n\u0000\t\n\r\r"+
-    "  ,,//==[[]]``||\u0002\u0000**//\u0001\u000009\u0002\u0000AZaz\u0005\u0000"+
-    "\"\"\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000EEee\u0002\u0000"+
-    "++--\u0001\u0000``\u000b\u0000\t\n\r\r  \"#,,//::<<>?\\\\||\u052a\u0000"+
-    "\u000e\u0001\u0000\u0000\u0000\u0000\u0010\u0001\u0000\u0000\u0000\u0000"+
-    "\u0012\u0001\u0000\u0000\u0000\u0000\u0014\u0001\u0000\u0000\u0000\u0000"+
-    "\u0016\u0001\u0000\u0000\u0000\u0000\u0018\u0001\u0000\u0000\u0000\u0000"+
-    "\u001a\u0001\u0000\u0000\u0000\u0000\u001c\u0001\u0000\u0000\u0000\u0000"+
-    "\u001e\u0001\u0000\u0000\u0000\u0000 \u0001\u0000\u0000\u0000\u0000\""+
-    "\u0001\u0000\u0000\u0000\u0000$\u0001\u0000\u0000\u0000\u0000&\u0001\u0000"+
-    "\u0000\u0000\u0000(\u0001\u0000\u0000\u0000\u0000*\u0001\u0000\u0000\u0000"+
-    "\u0000,\u0001\u0000\u0000\u0000\u0000.\u0001\u0000\u0000\u0000\u00000"+
-    "\u0001\u0000\u0000\u0000\u00002\u0001\u0000\u0000\u0000\u00004\u0001\u0000"+
-    "\u0000\u0000\u00006\u0001\u0000\u0000\u0000\u00008\u0001\u0000\u0000\u0000"+
-    "\u0000:\u0001\u0000\u0000\u0000\u0000>\u0001\u0000\u0000\u0000\u0001@"+
-    "\u0001\u0000\u0000\u0000\u0001B\u0001\u0000\u0000\u0000\u0001D\u0001\u0000"+
-    "\u0000\u0000\u0001F\u0001\u0000\u0000\u0000\u0001H\u0001\u0000\u0000\u0000"+
-    "\u0002J\u0001\u0000\u0000\u0000\u0002`\u0001\u0000\u0000\u0000\u0002b"+
-    "\u0001\u0000\u0000\u0000\u0002d\u0001\u0000\u0000\u0000\u0002f\u0001\u0000"+
-    "\u0000\u0000\u0002h\u0001\u0000\u0000\u0000\u0002j\u0001\u0000\u0000\u0000"+
-    "\u0002l\u0001\u0000\u0000\u0000\u0002n\u0001\u0000\u0000\u0000\u0002p"+
-    "\u0001\u0000\u0000\u0000\u0002r\u0001\u0000\u0000\u0000\u0002t\u0001\u0000"+
-    "\u0000\u0000\u0002v\u0001\u0000\u0000\u0000\u0002x\u0001\u0000\u0000\u0000"+
-    "\u0002z\u0001\u0000\u0000\u0000\u0002|\u0001\u0000\u0000\u0000\u0002~"+
-    "\u0001\u0000\u0000\u0000\u0002\u0080\u0001\u0000\u0000\u0000\u0002\u0082"+
+    "\u0099\u0001\u0099\u0001\u0099\u0001\u009a\u0001\u009a\u0001\u009a\u0001"+
+    "\u009a\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009c\u0001"+
+    "\u009c\u0001\u009c\u0001\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001"+
+    "\u009d\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009f\u0001"+
+    "\u009f\u0001\u009f\u0001\u009f\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001"+
+    "\u00a0\u0001\u00a0\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001"+
+    "\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a3\u0001"+
+    "\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001"+
+    "\u00a4\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001"+
+    "\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001"+
+    "\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7\u0001\u00a7\u0001"+
+    "\u00a7\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001"+
+    "\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00aa\u0001\u00aa\u0001"+
+    "\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001\u00ab\u0001\u00ac\u0001"+
+    "\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0004\u00ac\u052d\b\u00ac\u000b"+
+    "\u00ac\f\u00ac\u052e\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001"+
+    "\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001\u00af\u0001"+
+    "\u00af\u0001\u00af\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001"+
+    "\u00b0\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001"+
+    "\u00b1\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b3\u0001"+
+    "\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001"+
+    "\u00b4\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001"+
+    "\u00b5\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b7\u0001"+
+    "\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001"+
+    "\u00b8\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001"+
+    "\u00b9\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001"+
+    "\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001"+
+    "\u00bb\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0002"+
+    "\u0252\u02b7\u0000\u00bd\u0010\u0001\u0012\u0002\u0014\u0003\u0016\u0004"+
+    "\u0018\u0005\u001a\u0006\u001c\u0007\u001e\b \t\"\n$\u000b&\f(\r*\u000e"+
+    ",\u000f.\u00100\u00112\u00124\u00136\u00148\u0015:\u0016<\u0017>\u0018"+
+    "@\u0000B\u0019D\u0000F\u0000H\u001aJ\u001bL\u001cN\u001dP\u0000R\u0000"+
+    "T\u0000V\u0000X\u0000Z\u0000\\\u0000^\u0000`\u0000b\u0000d\u001ef\u001f"+
+    "h j!l\"n#p$r%t&v\'x(z)|*~+\u0080,\u0082-\u0084.\u0086/\u00880\u008a1\u008c"+
+    "2\u008e3\u00904\u00925\u00946\u00967\u00988\u009a9\u009c:\u009e;\u00a0"+
+    "<\u00a2=\u00a4>\u00a6?\u00a8@\u00aaA\u00acB\u00aeC\u00b0D\u00b2E\u00b4"+
+    "F\u00b6\u0000\u00b8G\u00baH\u00bcI\u00beJ\u00c0\u0000\u00c2\u0000\u00c4"+
+    "\u0000\u00c6\u0000\u00c8\u0000\u00ca\u0000\u00ccK\u00ce\u0000\u00d0L\u00d2"+
+    "M\u00d4N\u00d6\u0000\u00d8\u0000\u00da\u0000\u00dc\u0000\u00de\u0000\u00e0"+
+    "O\u00e2P\u00e4Q\u00e6R\u00e8\u0000\u00ea\u0000\u00ec\u0000\u00ee\u0000"+
+    "\u00f0S\u00f2\u0000\u00f4T\u00f6U\u00f8V\u00fa\u0000\u00fc\u0000\u00fe"+
+    "W\u0100X\u0102\u0000\u0104Y\u0106\u0000\u0108\u0000\u010aZ\u010c[\u010e"+
+    "\\\u0110\u0000\u0112\u0000\u0114\u0000\u0116\u0000\u0118\u0000\u011a\u0000"+
+    "\u011c\u0000\u011e]\u0120^\u0122_\u0124\u0000\u0126\u0000\u0128\u0000"+
+    "\u012a\u0000\u012c\u0000\u012e`\u0130a\u0132b\u0134\u0000\u0136\u0000"+
+    "\u0138\u0000\u013a\u0000\u013cc\u013ed\u0140e\u0142\u0000\u0144\u0000"+
+    "\u0146\u0000\u0148\u0000\u014af\u014cg\u014eh\u0150\u0000\u0152i\u0154"+
+    "j\u0156k\u0158l\u015a\u0000\u015cm\u015en\u0160o\u0162p\u0164\u0000\u0166"+
+    "q\u0168r\u016as\u016ct\u016eu\u0170\u0000\u0172\u0000\u0174v\u0176w\u0178"+
+    "x\u017a\u0000\u017cy\u017ez\u0180{\u0182\u0000\u0184\u0000\u0186\u0000"+
+    "\u0188\u0000\u0010\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t"+
+    "\n\u000b\f\r\u000e\u000f\r\u0006\u0000\t\n\r\r  //[[]]\u0002\u0000\n\n"+
+    "\r\r\u0003\u0000\t\n\r\r  \n\u0000\t\n\r\r  ,,//==[[]]``||\u0002\u0000"+
+    "**//\u0001\u000009\u0002\u0000AZaz\u0005\u0000\"\"\\\\nnrrtt\u0004\u0000"+
+    "\n\n\r\r\"\"\\\\\u0002\u0000EEee\u0002\u0000++--\u0001\u0000``\u000b\u0000"+
+    "\t\n\r\r  \"#,,//::<<>?\\\\||\u0593\u0000\u0010\u0001\u0000\u0000\u0000"+
+    "\u0000\u0012\u0001\u0000\u0000\u0000\u0000\u0014\u0001\u0000\u0000\u0000"+
+    "\u0000\u0016\u0001\u0000\u0000\u0000\u0000\u0018\u0001\u0000\u0000\u0000"+
+    "\u0000\u001a\u0001\u0000\u0000\u0000\u0000\u001c\u0001\u0000\u0000\u0000"+
+    "\u0000\u001e\u0001\u0000\u0000\u0000\u0000 \u0001\u0000\u0000\u0000\u0000"+
+    "\"\u0001\u0000\u0000\u0000\u0000$\u0001\u0000\u0000\u0000\u0000&\u0001"+
+    "\u0000\u0000\u0000\u0000(\u0001\u0000\u0000\u0000\u0000*\u0001\u0000\u0000"+
+    "\u0000\u0000,\u0001\u0000\u0000\u0000\u0000.\u0001\u0000\u0000\u0000\u0000"+
+    "0\u0001\u0000\u0000\u0000\u00002\u0001\u0000\u0000\u0000\u00004\u0001"+
+    "\u0000\u0000\u0000\u00006\u0001\u0000\u0000\u0000\u00008\u0001\u0000\u0000"+
+    "\u0000\u0000:\u0001\u0000\u0000\u0000\u0000<\u0001\u0000\u0000\u0000\u0000"+
+    ">\u0001\u0000\u0000\u0000\u0000B\u0001\u0000\u0000\u0000\u0001D\u0001"+
+    "\u0000\u0000\u0000\u0001F\u0001\u0000\u0000\u0000\u0001H\u0001\u0000\u0000"+
+    "\u0000\u0001J\u0001\u0000\u0000\u0000\u0001L\u0001\u0000\u0000\u0000\u0002"+
+    "N\u0001\u0000\u0000\u0000\u0002d\u0001\u0000\u0000\u0000\u0002f\u0001"+
+    "\u0000\u0000\u0000\u0002h\u0001\u0000\u0000\u0000\u0002j\u0001\u0000\u0000"+
+    "\u0000\u0002l\u0001\u0000\u0000\u0000\u0002n\u0001\u0000\u0000\u0000\u0002"+
+    "p\u0001\u0000\u0000\u0000\u0002r\u0001\u0000\u0000\u0000\u0002t\u0001"+
+    "\u0000\u0000\u0000\u0002v\u0001\u0000\u0000\u0000\u0002x\u0001\u0000\u0000"+
+    "\u0000\u0002z\u0001\u0000\u0000\u0000\u0002|\u0001\u0000\u0000\u0000\u0002"+
+    "~\u0001\u0000\u0000\u0000\u0002\u0080\u0001\u0000\u0000\u0000\u0002\u0082"+
     "\u0001\u0000\u0000\u0000\u0002\u0084\u0001\u0000\u0000\u0000\u0002\u0086"+
     "\u0001\u0000\u0000\u0000\u0002\u0088\u0001\u0000\u0000\u0000\u0002\u008a"+
     "\u0001\u0000\u0000\u0000\u0002\u008c\u0001\u0000\u0000\u0000\u0002\u008e"+
@@ -428,598 +459,651 @@ public EsqlBaseLexer(CharStream input) {
     "\u0001\u0000\u0000\u0000\u0002\u00a4\u0001\u0000\u0000\u0000\u0002\u00a6"+
     "\u0001\u0000\u0000\u0000\u0002\u00a8\u0001\u0000\u0000\u0000\u0002\u00aa"+
     "\u0001\u0000\u0000\u0000\u0002\u00ac\u0001\u0000\u0000\u0000\u0002\u00ae"+
-    "\u0001\u0000\u0000\u0000\u0002\u00b0\u0001\u0000\u0000\u0000\u0002\u00b4"+
-    "\u0001\u0000\u0000\u0000\u0002\u00b6\u0001\u0000\u0000\u0000\u0002\u00b8"+
-    "\u0001\u0000\u0000\u0000\u0002\u00ba\u0001\u0000\u0000\u0000\u0003\u00bc"+
-    "\u0001\u0000\u0000\u0000\u0003\u00be\u0001\u0000\u0000\u0000\u0003\u00c0"+
+    "\u0001\u0000\u0000\u0000\u0002\u00b0\u0001\u0000\u0000\u0000\u0002\u00b2"+
+    "\u0001\u0000\u0000\u0000\u0002\u00b4\u0001\u0000\u0000\u0000\u0002\u00b8"+
+    "\u0001\u0000\u0000\u0000\u0002\u00ba\u0001\u0000\u0000\u0000\u0002\u00bc"+
+    "\u0001\u0000\u0000\u0000\u0002\u00be\u0001\u0000\u0000\u0000\u0003\u00c0"+
     "\u0001\u0000\u0000\u0000\u0003\u00c2\u0001\u0000\u0000\u0000\u0003\u00c4"+
     "\u0001\u0000\u0000\u0000\u0003\u00c6\u0001\u0000\u0000\u0000\u0003\u00c8"+
     "\u0001\u0000\u0000\u0000\u0003\u00ca\u0001\u0000\u0000\u0000\u0003\u00cc"+
     "\u0001\u0000\u0000\u0000\u0003\u00ce\u0001\u0000\u0000\u0000\u0003\u00d0"+
-    "\u0001\u0000\u0000\u0000\u0004\u00d2\u0001\u0000\u0000\u0000\u0004\u00d4"+
-    "\u0001\u0000\u0000\u0000\u0004\u00d6\u0001\u0000\u0000\u0000\u0004\u00dc"+
-    "\u0001\u0000\u0000\u0000\u0004\u00de\u0001\u0000\u0000\u0000\u0004\u00e0"+
-    "\u0001\u0000\u0000\u0000\u0004\u00e2\u0001\u0000\u0000\u0000\u0005\u00e4"+
-    "\u0001\u0000\u0000\u0000\u0005\u00e6\u0001\u0000\u0000\u0000\u0005\u00e8"+
+    "\u0001\u0000\u0000\u0000\u0003\u00d2\u0001\u0000\u0000\u0000\u0003\u00d4"+
+    "\u0001\u0000\u0000\u0000\u0004\u00d6\u0001\u0000\u0000\u0000\u0004\u00d8"+
+    "\u0001\u0000\u0000\u0000\u0004\u00da\u0001\u0000\u0000\u0000\u0004\u00e0"+
+    "\u0001\u0000\u0000\u0000\u0004\u00e2\u0001\u0000\u0000\u0000\u0004\u00e4"+
+    "\u0001\u0000\u0000\u0000\u0004\u00e6\u0001\u0000\u0000\u0000\u0005\u00e8"+
     "\u0001\u0000\u0000\u0000\u0005\u00ea\u0001\u0000\u0000\u0000\u0005\u00ec"+
     "\u0001\u0000\u0000\u0000\u0005\u00ee\u0001\u0000\u0000\u0000\u0005\u00f0"+
     "\u0001\u0000\u0000\u0000\u0005\u00f2\u0001\u0000\u0000\u0000\u0005\u00f4"+
-    "\u0001\u0000\u0000\u0000\u0006\u00f6\u0001\u0000\u0000\u0000\u0006\u00f8"+
+    "\u0001\u0000\u0000\u0000\u0005\u00f6\u0001\u0000\u0000\u0000\u0005\u00f8"+
     "\u0001\u0000\u0000\u0000\u0006\u00fa\u0001\u0000\u0000\u0000\u0006\u00fc"+
-    "\u0001\u0000\u0000\u0000\u0006\u0100\u0001\u0000\u0000\u0000\u0006\u0102"+
+    "\u0001\u0000\u0000\u0000\u0006\u00fe\u0001\u0000\u0000\u0000\u0006\u0100"+
     "\u0001\u0000\u0000\u0000\u0006\u0104\u0001\u0000\u0000\u0000\u0006\u0106"+
     "\u0001\u0000\u0000\u0000\u0006\u0108\u0001\u0000\u0000\u0000\u0006\u010a"+
-    "\u0001\u0000\u0000\u0000\u0007\u010c\u0001\u0000\u0000\u0000\u0007\u010e"+
+    "\u0001\u0000\u0000\u0000\u0006\u010c\u0001\u0000\u0000\u0000\u0006\u010e"+
     "\u0001\u0000\u0000\u0000\u0007\u0110\u0001\u0000\u0000\u0000\u0007\u0112"+
     "\u0001\u0000\u0000\u0000\u0007\u0114\u0001\u0000\u0000\u0000\u0007\u0116"+
     "\u0001\u0000\u0000\u0000\u0007\u0118\u0001\u0000\u0000\u0000\u0007\u011a"+
     "\u0001\u0000\u0000\u0000\u0007\u011c\u0001\u0000\u0000\u0000\u0007\u011e"+
-    "\u0001\u0000\u0000\u0000\b\u0120\u0001\u0000\u0000\u0000\b\u0122\u0001"+
-    "\u0000\u0000\u0000\b\u0124\u0001\u0000\u0000\u0000\b\u0126\u0001\u0000"+
-    "\u0000\u0000\b\u0128\u0001\u0000\u0000\u0000\b\u012a\u0001\u0000\u0000"+
-    "\u0000\b\u012c\u0001\u0000\u0000\u0000\t\u012e\u0001\u0000\u0000\u0000"+
-    "\t\u0130\u0001\u0000\u0000\u0000\t\u0132\u0001\u0000\u0000\u0000\t\u0134"+
-    "\u0001\u0000\u0000\u0000\t\u0136\u0001\u0000\u0000\u0000\n\u0138\u0001"+
-    "\u0000\u0000\u0000\n\u013a\u0001\u0000\u0000\u0000\n\u013c\u0001\u0000"+
-    "\u0000\u0000\n\u013e\u0001\u0000\u0000\u0000\n\u0140\u0001\u0000\u0000"+
-    "\u0000\u000b\u0142\u0001\u0000\u0000\u0000\u000b\u0144\u0001\u0000\u0000"+
-    "\u0000\u000b\u0146\u0001\u0000\u0000\u0000\u000b\u0148\u0001\u0000\u0000"+
-    "\u0000\u000b\u014a\u0001\u0000\u0000\u0000\u000b\u014c\u0001\u0000\u0000"+
-    "\u0000\f\u014e\u0001\u0000\u0000\u0000\f\u0150\u0001\u0000\u0000\u0000"+
-    "\f\u0152\u0001\u0000\u0000\u0000\f\u0154\u0001\u0000\u0000\u0000\f\u0156"+
-    "\u0001\u0000\u0000\u0000\r\u0158\u0001\u0000\u0000\u0000\r\u015a\u0001"+
-    "\u0000\u0000\u0000\r\u015c\u0001\u0000\u0000\u0000\r\u015e\u0001\u0000"+
-    "\u0000\u0000\r\u0160\u0001\u0000\u0000\u0000\r\u0162\u0001\u0000\u0000"+
+    "\u0001\u0000\u0000\u0000\u0007\u0120\u0001\u0000\u0000\u0000\u0007\u0122"+
+    "\u0001\u0000\u0000\u0000\b\u0124\u0001\u0000\u0000\u0000\b\u0126\u0001"+
+    "\u0000\u0000\u0000\b\u0128\u0001\u0000\u0000\u0000\b\u012a\u0001\u0000"+
+    "\u0000\u0000\b\u012c\u0001\u0000\u0000\u0000\b\u012e\u0001\u0000\u0000"+
+    "\u0000\b\u0130\u0001\u0000\u0000\u0000\b\u0132\u0001\u0000\u0000\u0000"+
+    "\t\u0134\u0001\u0000\u0000\u0000\t\u0136\u0001\u0000\u0000\u0000\t\u0138"+
+    "\u0001\u0000\u0000\u0000\t\u013a\u0001\u0000\u0000\u0000\t\u013c\u0001"+
+    "\u0000\u0000\u0000\t\u013e\u0001\u0000\u0000\u0000\t\u0140\u0001\u0000"+
+    "\u0000\u0000\n\u0142\u0001\u0000\u0000\u0000\n\u0144\u0001\u0000\u0000"+
+    "\u0000\n\u0146\u0001\u0000\u0000\u0000\n\u0148\u0001\u0000\u0000\u0000"+
+    "\n\u014a\u0001\u0000\u0000\u0000\n\u014c\u0001\u0000\u0000\u0000\n\u014e"+
+    "\u0001\u0000\u0000\u0000\u000b\u0150\u0001\u0000\u0000\u0000\u000b\u0152"+
+    "\u0001\u0000\u0000\u0000\u000b\u0154\u0001\u0000\u0000\u0000\u000b\u0156"+
+    "\u0001\u0000\u0000\u0000\u000b\u0158\u0001\u0000\u0000\u0000\f\u015a\u0001"+
+    "\u0000\u0000\u0000\f\u015c\u0001\u0000\u0000\u0000\f\u015e\u0001\u0000"+
+    "\u0000\u0000\f\u0160\u0001\u0000\u0000\u0000\f\u0162\u0001\u0000\u0000"+
     "\u0000\r\u0164\u0001\u0000\u0000\u0000\r\u0166\u0001\u0000\u0000\u0000"+
-    "\u000e\u0168\u0001\u0000\u0000\u0000\u0010\u0172\u0001\u0000\u0000\u0000"+
-    "\u0012\u0179\u0001\u0000\u0000\u0000\u0014\u0182\u0001\u0000\u0000\u0000"+
-    "\u0016\u0189\u0001\u0000\u0000\u0000\u0018\u0193\u0001\u0000\u0000\u0000"+
-    "\u001a\u019a\u0001\u0000\u0000\u0000\u001c\u01a1\u0001\u0000\u0000\u0000"+
-    "\u001e\u01af\u0001\u0000\u0000\u0000 \u01b6\u0001\u0000\u0000\u0000\""+
-    "\u01be\u0001\u0000\u0000\u0000$\u01c5\u0001\u0000\u0000\u0000&\u01cf\u0001"+
-    "\u0000\u0000\u0000(\u01db\u0001\u0000\u0000\u0000*\u01e4\u0001\u0000\u0000"+
-    "\u0000,\u01ea\u0001\u0000\u0000\u0000.\u01f1\u0001\u0000\u0000\u00000"+
-    "\u01f8\u0001\u0000\u0000\u00002\u0200\u0001\u0000\u0000\u00004\u0209\u0001"+
-    "\u0000\u0000\u00006\u020f\u0001\u0000\u0000\u00008\u0220\u0001\u0000\u0000"+
-    "\u0000:\u0230\u0001\u0000\u0000\u0000<\u0239\u0001\u0000\u0000\u0000>"+
-    "\u023c\u0001\u0000\u0000\u0000@\u0240\u0001\u0000\u0000\u0000B\u0245\u0001"+
-    "\u0000\u0000\u0000D\u024a\u0001\u0000\u0000\u0000F\u024e\u0001\u0000\u0000"+
-    "\u0000H\u0252\u0001\u0000\u0000\u0000J\u0256\u0001\u0000\u0000\u0000L"+
-    "\u025a\u0001\u0000\u0000\u0000N\u025c\u0001\u0000\u0000\u0000P\u025e\u0001"+
-    "\u0000\u0000\u0000R\u0261\u0001\u0000\u0000\u0000T\u0263\u0001\u0000\u0000"+
-    "\u0000V\u026c\u0001\u0000\u0000\u0000X\u026e\u0001\u0000\u0000\u0000Z"+
-    "\u0273\u0001\u0000\u0000\u0000\\\u0275\u0001\u0000\u0000\u0000^\u027a"+
-    "\u0001\u0000\u0000\u0000`\u0299\u0001\u0000\u0000\u0000b\u029c\u0001\u0000"+
-    "\u0000\u0000d\u02ca\u0001\u0000\u0000\u0000f\u02cc\u0001\u0000\u0000\u0000"+
-    "h\u02cf\u0001\u0000\u0000\u0000j\u02d3\u0001\u0000\u0000\u0000l\u02d7"+
-    "\u0001\u0000\u0000\u0000n\u02d9\u0001\u0000\u0000\u0000p\u02dc\u0001\u0000"+
-    "\u0000\u0000r\u02de\u0001\u0000\u0000\u0000t\u02e3\u0001\u0000\u0000\u0000"+
-    "v\u02e5\u0001\u0000\u0000\u0000x\u02eb\u0001\u0000\u0000\u0000z\u02f1"+
-    "\u0001\u0000\u0000\u0000|\u02f6\u0001\u0000\u0000\u0000~\u02f8\u0001\u0000"+
-    "\u0000\u0000\u0080\u02fb\u0001\u0000\u0000\u0000\u0082\u02fe\u0001\u0000"+
-    "\u0000\u0000\u0084\u0303\u0001\u0000\u0000\u0000\u0086\u0307\u0001\u0000"+
-    "\u0000\u0000\u0088\u030c\u0001\u0000\u0000\u0000\u008a\u0312\u0001\u0000"+
-    "\u0000\u0000\u008c\u0315\u0001\u0000\u0000\u0000\u008e\u0317\u0001\u0000"+
-    "\u0000\u0000\u0090\u031d\u0001\u0000\u0000\u0000\u0092\u031f\u0001\u0000"+
-    "\u0000\u0000\u0094\u0324\u0001\u0000\u0000\u0000\u0096\u0327\u0001\u0000"+
-    "\u0000\u0000\u0098\u032a\u0001\u0000\u0000\u0000\u009a\u032d\u0001\u0000"+
-    "\u0000\u0000\u009c\u032f\u0001\u0000\u0000\u0000\u009e\u0332\u0001\u0000"+
-    "\u0000\u0000\u00a0\u0334\u0001\u0000\u0000\u0000\u00a2\u0337\u0001\u0000"+
-    "\u0000\u0000\u00a4\u0339\u0001\u0000\u0000\u0000\u00a6\u033b\u0001\u0000"+
-    "\u0000\u0000\u00a8\u033d\u0001\u0000\u0000\u0000\u00aa\u033f\u0001\u0000"+
-    "\u0000\u0000\u00ac\u0341\u0001\u0000\u0000\u0000\u00ae\u0346\u0001\u0000"+
-    "\u0000\u0000\u00b0\u035b\u0001\u0000\u0000\u0000\u00b2\u035d\u0001\u0000"+
-    "\u0000\u0000\u00b4\u0365\u0001\u0000\u0000\u0000\u00b6\u0367\u0001\u0000"+
-    "\u0000\u0000\u00b8\u036b\u0001\u0000\u0000\u0000\u00ba\u036f\u0001\u0000"+
-    "\u0000\u0000\u00bc\u0373\u0001\u0000\u0000\u0000\u00be\u0378\u0001\u0000"+
-    "\u0000\u0000\u00c0\u037c\u0001\u0000\u0000\u0000\u00c2\u0380\u0001\u0000"+
-    "\u0000\u0000\u00c4\u0384\u0001\u0000\u0000\u0000\u00c6\u0388\u0001\u0000"+
-    "\u0000\u0000\u00c8\u038c\u0001\u0000\u0000\u0000\u00ca\u0395\u0001\u0000"+
-    "\u0000\u0000\u00cc\u0399\u0001\u0000\u0000\u0000\u00ce\u039d\u0001\u0000"+
-    "\u0000\u0000\u00d0\u03a1\u0001\u0000\u0000\u0000\u00d2\u03a5\u0001\u0000"+
-    "\u0000\u0000\u00d4\u03aa\u0001\u0000\u0000\u0000\u00d6\u03ae\u0001\u0000"+
-    "\u0000\u0000\u00d8\u03b6\u0001\u0000\u0000\u0000\u00da\u03cb\u0001\u0000"+
-    "\u0000\u0000\u00dc\u03cf\u0001\u0000\u0000\u0000\u00de\u03d3\u0001\u0000"+
-    "\u0000\u0000\u00e0\u03d7\u0001\u0000\u0000\u0000\u00e2\u03db\u0001\u0000"+
-    "\u0000\u0000\u00e4\u03df\u0001\u0000\u0000\u0000\u00e6\u03e4\u0001\u0000"+
-    "\u0000\u0000\u00e8\u03e8\u0001\u0000\u0000\u0000\u00ea\u03ec\u0001\u0000"+
-    "\u0000\u0000\u00ec\u03f0\u0001\u0000\u0000\u0000\u00ee\u03f3\u0001\u0000"+
-    "\u0000\u0000\u00f0\u03f7\u0001\u0000\u0000\u0000\u00f2\u03fb\u0001\u0000"+
-    "\u0000\u0000\u00f4\u03ff\u0001\u0000\u0000\u0000\u00f6\u0403\u0001\u0000"+
-    "\u0000\u0000\u00f8\u0408\u0001\u0000\u0000\u0000\u00fa\u040d\u0001\u0000"+
-    "\u0000\u0000\u00fc\u0412\u0001\u0000\u0000\u0000\u00fe\u0419\u0001\u0000"+
-    "\u0000\u0000\u0100\u0422\u0001\u0000\u0000\u0000\u0102\u0429\u0001\u0000"+
-    "\u0000\u0000\u0104\u042d\u0001\u0000\u0000\u0000\u0106\u0431\u0001\u0000"+
-    "\u0000\u0000\u0108\u0435\u0001\u0000\u0000\u0000\u010a\u0439\u0001\u0000"+
-    "\u0000\u0000\u010c\u043d\u0001\u0000\u0000\u0000\u010e\u0443\u0001\u0000"+
-    "\u0000\u0000\u0110\u0447\u0001\u0000\u0000\u0000\u0112\u044b\u0001\u0000"+
-    "\u0000\u0000\u0114\u044f\u0001\u0000\u0000\u0000\u0116\u0453\u0001\u0000"+
-    "\u0000\u0000\u0118\u0457\u0001\u0000\u0000\u0000\u011a\u045b\u0001\u0000"+
-    "\u0000\u0000\u011c\u045f\u0001\u0000\u0000\u0000\u011e\u0463\u0001\u0000"+
-    "\u0000\u0000\u0120\u0467\u0001\u0000\u0000\u0000\u0122\u046c\u0001\u0000"+
-    "\u0000\u0000\u0124\u0470\u0001\u0000\u0000\u0000\u0126\u0474\u0001\u0000"+
-    "\u0000\u0000\u0128\u0478\u0001\u0000\u0000\u0000\u012a\u047c\u0001\u0000"+
-    "\u0000\u0000\u012c\u0480\u0001\u0000\u0000\u0000\u012e\u0484\u0001\u0000"+
-    "\u0000\u0000\u0130\u0489\u0001\u0000\u0000\u0000\u0132\u048e\u0001\u0000"+
-    "\u0000\u0000\u0134\u0492\u0001\u0000\u0000\u0000\u0136\u0496\u0001\u0000"+
-    "\u0000\u0000\u0138\u049a\u0001\u0000\u0000\u0000\u013a\u049f\u0001\u0000"+
-    "\u0000\u0000\u013c\u04a9\u0001\u0000\u0000\u0000\u013e\u04ad\u0001\u0000"+
-    "\u0000\u0000\u0140\u04b1\u0001\u0000\u0000\u0000\u0142\u04b5\u0001\u0000"+
-    "\u0000\u0000\u0144\u04ba\u0001\u0000\u0000\u0000\u0146\u04c1\u0001\u0000"+
-    "\u0000\u0000\u0148\u04c5\u0001\u0000\u0000\u0000\u014a\u04c9\u0001\u0000"+
-    "\u0000\u0000\u014c\u04cd\u0001\u0000\u0000\u0000\u014e\u04d1\u0001\u0000"+
-    "\u0000\u0000\u0150\u04d6\u0001\u0000\u0000\u0000\u0152\u04dc\u0001\u0000"+
-    "\u0000\u0000\u0154\u04e0\u0001\u0000\u0000\u0000\u0156\u04e4\u0001\u0000"+
-    "\u0000\u0000\u0158\u04e8\u0001\u0000\u0000\u0000\u015a\u04ee\u0001\u0000"+
-    "\u0000\u0000\u015c\u04f2\u0001\u0000\u0000\u0000\u015e\u04f6\u0001\u0000"+
-    "\u0000\u0000\u0160\u04fa\u0001\u0000\u0000\u0000\u0162\u0500\u0001\u0000"+
-    "\u0000\u0000\u0164\u0506\u0001\u0000\u0000\u0000\u0166\u050c\u0001\u0000"+
-    "\u0000\u0000\u0168\u0169\u0005d\u0000\u0000\u0169\u016a\u0005i\u0000\u0000"+
-    "\u016a\u016b\u0005s\u0000\u0000\u016b\u016c\u0005s\u0000\u0000\u016c\u016d"+
-    "\u0005e\u0000\u0000\u016d\u016e\u0005c\u0000\u0000\u016e\u016f\u0005t"+
-    "\u0000\u0000\u016f\u0170\u0001\u0000\u0000\u0000\u0170\u0171\u0006\u0000"+
-    "\u0000\u0000\u0171\u000f\u0001\u0000\u0000\u0000\u0172\u0173\u0005d\u0000"+
-    "\u0000\u0173\u0174\u0005r\u0000\u0000\u0174\u0175\u0005o\u0000\u0000\u0175"+
-    "\u0176\u0005p\u0000\u0000\u0176\u0177\u0001\u0000\u0000\u0000\u0177\u0178"+
-    "\u0006\u0001\u0001\u0000\u0178\u0011\u0001\u0000\u0000\u0000\u0179\u017a"+
-    "\u0005e\u0000\u0000\u017a\u017b\u0005n\u0000\u0000\u017b\u017c\u0005r"+
-    "\u0000\u0000\u017c\u017d\u0005i\u0000\u0000\u017d\u017e\u0005c\u0000\u0000"+
-    "\u017e\u017f\u0005h\u0000\u0000\u017f\u0180\u0001\u0000\u0000\u0000\u0180"+
-    "\u0181\u0006\u0002\u0002\u0000\u0181\u0013\u0001\u0000\u0000\u0000\u0182"+
-    "\u0183\u0005e\u0000\u0000\u0183\u0184\u0005v\u0000\u0000\u0184\u0185\u0005"+
-    "a\u0000\u0000\u0185\u0186\u0005l\u0000\u0000\u0186\u0187\u0001\u0000\u0000"+
-    "\u0000\u0187\u0188\u0006\u0003\u0000\u0000\u0188\u0015\u0001\u0000\u0000"+
-    "\u0000\u0189\u018a\u0005e\u0000\u0000\u018a\u018b\u0005x\u0000\u0000\u018b"+
-    "\u018c\u0005p\u0000\u0000\u018c\u018d\u0005l\u0000\u0000\u018d\u018e\u0005"+
-    "a\u0000\u0000\u018e\u018f\u0005i\u0000\u0000\u018f\u0190\u0005n\u0000"+
-    "\u0000\u0190\u0191\u0001\u0000\u0000\u0000\u0191\u0192\u0006\u0004\u0003"+
-    "\u0000\u0192\u0017\u0001\u0000\u0000\u0000\u0193\u0194\u0005f\u0000\u0000"+
-    "\u0194\u0195\u0005r\u0000\u0000\u0195\u0196\u0005o\u0000\u0000\u0196\u0197"+
-    "\u0005m\u0000\u0000\u0197\u0198\u0001\u0000\u0000\u0000\u0198\u0199\u0006"+
-    "\u0005\u0004\u0000\u0199\u0019\u0001\u0000\u0000\u0000\u019a\u019b\u0005"+
-    "g\u0000\u0000\u019b\u019c\u0005r\u0000\u0000\u019c\u019d\u0005o\u0000"+
-    "\u0000\u019d\u019e\u0005k\u0000\u0000\u019e\u019f\u0001\u0000\u0000\u0000"+
-    "\u019f\u01a0\u0006\u0006\u0000\u0000\u01a0\u001b\u0001\u0000\u0000\u0000"+
-    "\u01a1\u01a2\u0005i\u0000\u0000\u01a2\u01a3\u0005n\u0000\u0000\u01a3\u01a4"+
-    "\u0005l\u0000\u0000\u01a4\u01a5\u0005i\u0000\u0000\u01a5\u01a6\u0005n"+
-    "\u0000\u0000\u01a6\u01a7\u0005e\u0000\u0000\u01a7\u01a8\u0005s\u0000\u0000"+
-    "\u01a8\u01a9\u0005t\u0000\u0000\u01a9\u01aa\u0005a\u0000\u0000\u01aa\u01ab"+
-    "\u0005t\u0000\u0000\u01ab\u01ac\u0005s\u0000\u0000\u01ac\u01ad\u0001\u0000"+
-    "\u0000\u0000\u01ad\u01ae\u0006\u0007\u0000\u0000\u01ae\u001d\u0001\u0000"+
-    "\u0000\u0000\u01af\u01b0\u0005k\u0000\u0000\u01b0\u01b1\u0005e\u0000\u0000"+
-    "\u01b1\u01b2\u0005e\u0000\u0000\u01b2\u01b3\u0005p\u0000\u0000\u01b3\u01b4"+
-    "\u0001\u0000\u0000\u0000\u01b4\u01b5\u0006\b\u0001\u0000\u01b5\u001f\u0001"+
-    "\u0000\u0000\u0000\u01b6\u01b7\u0005l\u0000\u0000\u01b7\u01b8\u0005i\u0000"+
-    "\u0000\u01b8\u01b9\u0005m\u0000\u0000\u01b9\u01ba\u0005i\u0000\u0000\u01ba"+
-    "\u01bb\u0005t\u0000\u0000\u01bb\u01bc\u0001\u0000\u0000\u0000\u01bc\u01bd"+
-    "\u0006\t\u0000\u0000\u01bd!\u0001\u0000\u0000\u0000\u01be\u01bf\u0005"+
-    "m\u0000\u0000\u01bf\u01c0\u0005e\u0000\u0000\u01c0\u01c1\u0005t\u0000"+
-    "\u0000\u01c1\u01c2\u0005a\u0000\u0000\u01c2\u01c3\u0001\u0000\u0000\u0000"+
-    "\u01c3\u01c4\u0006\n\u0005\u0000\u01c4#\u0001\u0000\u0000\u0000\u01c5"+
-    "\u01c6\u0005m\u0000\u0000\u01c6\u01c7\u0005e\u0000\u0000\u01c7\u01c8\u0005"+
-    "t\u0000\u0000\u01c8\u01c9\u0005r\u0000\u0000\u01c9\u01ca\u0005i\u0000"+
-    "\u0000\u01ca\u01cb\u0005c\u0000\u0000\u01cb\u01cc\u0005s\u0000\u0000\u01cc"+
-    "\u01cd\u0001\u0000\u0000\u0000\u01cd\u01ce\u0006\u000b\u0006\u0000\u01ce"+
-    "%\u0001\u0000\u0000\u0000\u01cf\u01d0\u0005m\u0000\u0000\u01d0\u01d1\u0005"+
-    "v\u0000\u0000\u01d1\u01d2\u0005_\u0000\u0000\u01d2\u01d3\u0005e\u0000"+
-    "\u0000\u01d3\u01d4\u0005x\u0000\u0000\u01d4\u01d5\u0005p\u0000\u0000\u01d5"+
-    "\u01d6\u0005a\u0000\u0000\u01d6\u01d7\u0005n\u0000\u0000\u01d7\u01d8\u0005"+
-    "d\u0000\u0000\u01d8\u01d9\u0001\u0000\u0000\u0000\u01d9\u01da\u0006\f"+
-    "\u0007\u0000\u01da\'\u0001\u0000\u0000\u0000\u01db\u01dc\u0005r\u0000"+
-    "\u0000\u01dc\u01dd\u0005e\u0000\u0000\u01dd\u01de\u0005n\u0000\u0000\u01de"+
-    "\u01df\u0005a\u0000\u0000\u01df\u01e0\u0005m\u0000\u0000\u01e0\u01e1\u0005"+
-    "e\u0000\u0000\u01e1\u01e2\u0001\u0000\u0000\u0000\u01e2\u01e3\u0006\r"+
-    "\b\u0000\u01e3)\u0001\u0000\u0000\u0000\u01e4\u01e5\u0005r\u0000\u0000"+
-    "\u01e5\u01e6\u0005o\u0000\u0000\u01e6\u01e7\u0005w\u0000\u0000\u01e7\u01e8"+
-    "\u0001\u0000\u0000\u0000\u01e8\u01e9\u0006\u000e\u0000\u0000\u01e9+\u0001"+
-    "\u0000\u0000\u0000\u01ea\u01eb\u0005s\u0000\u0000\u01eb\u01ec\u0005h\u0000"+
-    "\u0000\u01ec\u01ed\u0005o\u0000\u0000\u01ed\u01ee\u0005w\u0000\u0000\u01ee"+
-    "\u01ef\u0001\u0000\u0000\u0000\u01ef\u01f0\u0006\u000f\t\u0000\u01f0-"+
-    "\u0001\u0000\u0000\u0000\u01f1\u01f2\u0005s\u0000\u0000\u01f2\u01f3\u0005"+
-    "o\u0000\u0000\u01f3\u01f4\u0005r\u0000\u0000\u01f4\u01f5\u0005t\u0000"+
-    "\u0000\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u01f7\u0006\u0010\u0000"+
-    "\u0000\u01f7/\u0001\u0000\u0000\u0000\u01f8\u01f9\u0005s\u0000\u0000\u01f9"+
-    "\u01fa\u0005t\u0000\u0000\u01fa\u01fb\u0005a\u0000\u0000\u01fb\u01fc\u0005"+
-    "t\u0000\u0000\u01fc\u01fd\u0005s\u0000\u0000\u01fd\u01fe\u0001\u0000\u0000"+
-    "\u0000\u01fe\u01ff\u0006\u0011\u0000\u0000\u01ff1\u0001\u0000\u0000\u0000"+
-    "\u0200\u0201\u0005w\u0000\u0000\u0201\u0202\u0005h\u0000\u0000\u0202\u0203"+
-    "\u0005e\u0000\u0000\u0203\u0204\u0005r\u0000\u0000\u0204\u0205\u0005e"+
-    "\u0000\u0000\u0205\u0206\u0001\u0000\u0000\u0000\u0206\u0207\u0006\u0012"+
-    "\u0000\u0000\u02073\u0001\u0000\u0000\u0000\u0208\u020a\b\u0000\u0000"+
-    "\u0000\u0209\u0208\u0001\u0000\u0000\u0000\u020a\u020b\u0001\u0000\u0000"+
-    "\u0000\u020b\u0209\u0001\u0000\u0000\u0000\u020b\u020c\u0001\u0000\u0000"+
-    "\u0000\u020c\u020d\u0001\u0000\u0000\u0000\u020d\u020e\u0006\u0013\u0000"+
-    "\u0000\u020e5\u0001\u0000\u0000\u0000\u020f\u0210\u0005/\u0000\u0000\u0210"+
-    "\u0211\u0005/\u0000\u0000\u0211\u0215\u0001\u0000\u0000\u0000\u0212\u0214"+
-    "\b\u0001\u0000\u0000\u0213\u0212\u0001\u0000\u0000\u0000\u0214\u0217\u0001"+
-    "\u0000\u0000\u0000\u0215\u0213\u0001\u0000\u0000\u0000\u0215\u0216\u0001"+
-    "\u0000\u0000\u0000\u0216\u0219\u0001\u0000\u0000\u0000\u0217\u0215\u0001"+
-    "\u0000\u0000\u0000\u0218\u021a\u0005\r\u0000\u0000\u0219\u0218\u0001\u0000"+
-    "\u0000\u0000\u0219\u021a\u0001\u0000\u0000\u0000\u021a\u021c\u0001\u0000"+
-    "\u0000\u0000\u021b\u021d\u0005\n\u0000\u0000\u021c\u021b\u0001\u0000\u0000"+
-    "\u0000\u021c\u021d\u0001\u0000\u0000\u0000\u021d\u021e\u0001\u0000\u0000"+
-    "\u0000\u021e\u021f\u0006\u0014\n\u0000\u021f7\u0001\u0000\u0000\u0000"+
-    "\u0220\u0221\u0005/\u0000\u0000\u0221\u0222\u0005*\u0000\u0000\u0222\u0227"+
-    "\u0001\u0000\u0000\u0000\u0223\u0226\u00038\u0015\u0000\u0224\u0226\t"+
-    "\u0000\u0000\u0000\u0225\u0223\u0001\u0000\u0000\u0000\u0225\u0224\u0001"+
-    "\u0000\u0000\u0000\u0226\u0229\u0001\u0000\u0000\u0000\u0227\u0228\u0001"+
-    "\u0000\u0000\u0000\u0227\u0225\u0001\u0000\u0000\u0000\u0228\u022a\u0001"+
-    "\u0000\u0000\u0000\u0229\u0227\u0001\u0000\u0000\u0000\u022a\u022b\u0005"+
-    "*\u0000\u0000\u022b\u022c\u0005/\u0000\u0000\u022c\u022d\u0001\u0000\u0000"+
-    "\u0000\u022d\u022e\u0006\u0015\n\u0000\u022e9\u0001\u0000\u0000\u0000"+
-    "\u022f\u0231\u0007\u0002\u0000\u0000\u0230\u022f\u0001\u0000\u0000\u0000"+
-    "\u0231\u0232\u0001\u0000\u0000\u0000\u0232\u0230\u0001\u0000\u0000\u0000"+
-    "\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234\u0001\u0000\u0000\u0000"+
-    "\u0234\u0235\u0006\u0016\n\u0000\u0235;\u0001\u0000\u0000\u0000\u0236"+
-    "\u023a\b\u0003\u0000\u0000\u0237\u0238\u0005/\u0000\u0000\u0238\u023a"+
-    "\b\u0004\u0000\u0000\u0239\u0236\u0001\u0000\u0000\u0000\u0239\u0237\u0001"+
-    "\u0000\u0000\u0000\u023a=\u0001\u0000\u0000\u0000\u023b\u023d\u0003<\u0017"+
-    "\u0000\u023c\u023b\u0001\u0000\u0000\u0000\u023d\u023e\u0001\u0000\u0000"+
-    "\u0000\u023e\u023c\u0001\u0000\u0000\u0000\u023e\u023f\u0001\u0000\u0000"+
-    "\u0000\u023f?\u0001\u0000\u0000\u0000\u0240\u0241\u0003\u00acO\u0000\u0241"+
-    "\u0242\u0001\u0000\u0000\u0000\u0242\u0243\u0006\u0019\u000b\u0000\u0243"+
-    "\u0244\u0006\u0019\f\u0000\u0244A\u0001\u0000\u0000\u0000\u0245\u0246"+
-    "\u0003J\u001e\u0000\u0246\u0247\u0001\u0000\u0000\u0000\u0247\u0248\u0006"+
-    "\u001a\r\u0000\u0248\u0249\u0006\u001a\u000e\u0000\u0249C\u0001\u0000"+
-    "\u0000\u0000\u024a\u024b\u0003:\u0016\u0000\u024b\u024c\u0001\u0000\u0000"+
-    "\u0000\u024c\u024d\u0006\u001b\n\u0000\u024dE\u0001\u0000\u0000\u0000"+
-    "\u024e\u024f\u00036\u0014\u0000\u024f\u0250\u0001\u0000\u0000\u0000\u0250"+
-    "\u0251\u0006\u001c\n\u0000\u0251G\u0001\u0000\u0000\u0000\u0252\u0253"+
-    "\u00038\u0015\u0000\u0253\u0254\u0001\u0000\u0000\u0000\u0254\u0255\u0006"+
-    "\u001d\n\u0000\u0255I\u0001\u0000\u0000\u0000\u0256\u0257\u0005|\u0000"+
-    "\u0000\u0257\u0258\u0001\u0000\u0000\u0000\u0258\u0259\u0006\u001e\u000e"+
-    "\u0000\u0259K\u0001\u0000\u0000\u0000\u025a\u025b\u0007\u0005\u0000\u0000"+
-    "\u025bM\u0001\u0000\u0000\u0000\u025c\u025d\u0007\u0006\u0000\u0000\u025d"+
-    "O\u0001\u0000\u0000\u0000\u025e\u025f\u0005\\\u0000\u0000\u025f\u0260"+
-    "\u0007\u0007\u0000\u0000\u0260Q\u0001\u0000\u0000\u0000\u0261\u0262\b"+
-    "\b\u0000\u0000\u0262S\u0001\u0000\u0000\u0000\u0263\u0265\u0007\t\u0000"+
-    "\u0000\u0264\u0266\u0007\n\u0000\u0000\u0265\u0264\u0001\u0000\u0000\u0000"+
-    "\u0265\u0266\u0001\u0000\u0000\u0000\u0266\u0268\u0001\u0000\u0000\u0000"+
-    "\u0267\u0269\u0003L\u001f\u0000\u0268\u0267\u0001\u0000\u0000\u0000\u0269"+
-    "\u026a\u0001\u0000\u0000\u0000\u026a\u0268\u0001\u0000\u0000\u0000\u026a"+
-    "\u026b\u0001\u0000\u0000\u0000\u026bU\u0001\u0000\u0000\u0000\u026c\u026d"+
-    "\u0005@\u0000\u0000\u026dW\u0001\u0000\u0000\u0000\u026e\u026f\u0005`"+
-    "\u0000\u0000\u026fY\u0001\u0000\u0000\u0000\u0270\u0274\b\u000b\u0000"+
-    "\u0000\u0271\u0272\u0005`\u0000\u0000\u0272\u0274\u0005`\u0000\u0000\u0273"+
-    "\u0270\u0001\u0000\u0000\u0000\u0273\u0271\u0001\u0000\u0000\u0000\u0274"+
-    "[\u0001\u0000\u0000\u0000\u0275\u0276\u0005_\u0000\u0000\u0276]\u0001"+
-    "\u0000\u0000\u0000\u0277\u027b\u0003N \u0000\u0278\u027b\u0003L\u001f"+
-    "\u0000\u0279\u027b\u0003\\\'\u0000\u027a\u0277\u0001\u0000\u0000\u0000"+
-    "\u027a\u0278\u0001\u0000\u0000\u0000\u027a\u0279\u0001\u0000\u0000\u0000"+
-    "\u027b_\u0001\u0000\u0000\u0000\u027c\u0281\u0005\"\u0000\u0000\u027d"+
-    "\u0280\u0003P!\u0000\u027e\u0280\u0003R\"\u0000\u027f\u027d\u0001\u0000"+
-    "\u0000\u0000\u027f\u027e\u0001\u0000\u0000\u0000\u0280\u0283\u0001\u0000"+
-    "\u0000\u0000\u0281\u027f\u0001\u0000\u0000\u0000\u0281\u0282\u0001\u0000"+
-    "\u0000\u0000\u0282\u0284\u0001\u0000\u0000\u0000\u0283\u0281\u0001\u0000"+
-    "\u0000\u0000\u0284\u029a\u0005\"\u0000\u0000\u0285\u0286\u0005\"\u0000"+
-    "\u0000\u0286\u0287\u0005\"\u0000\u0000\u0287\u0288\u0005\"\u0000\u0000"+
-    "\u0288\u028c\u0001\u0000\u0000\u0000\u0289\u028b\b\u0001\u0000\u0000\u028a"+
-    "\u0289\u0001\u0000\u0000\u0000\u028b\u028e\u0001\u0000\u0000\u0000\u028c"+
-    "\u028d\u0001\u0000\u0000\u0000\u028c\u028a\u0001\u0000\u0000\u0000\u028d"+
-    "\u028f\u0001\u0000\u0000\u0000\u028e\u028c\u0001\u0000\u0000\u0000\u028f"+
-    "\u0290\u0005\"\u0000\u0000\u0290\u0291\u0005\"\u0000\u0000\u0291\u0292"+
-    "\u0005\"\u0000\u0000\u0292\u0294\u0001\u0000\u0000\u0000\u0293\u0295\u0005"+
-    "\"\u0000\u0000\u0294\u0293\u0001\u0000\u0000\u0000\u0294\u0295\u0001\u0000"+
-    "\u0000\u0000\u0295\u0297\u0001\u0000\u0000\u0000\u0296\u0298\u0005\"\u0000"+
-    "\u0000\u0297\u0296\u0001\u0000\u0000\u0000\u0297\u0298\u0001\u0000\u0000"+
-    "\u0000\u0298\u029a\u0001\u0000\u0000\u0000\u0299\u027c\u0001\u0000\u0000"+
-    "\u0000\u0299\u0285\u0001\u0000\u0000\u0000\u029aa\u0001\u0000\u0000\u0000"+
-    "\u029b\u029d\u0003L\u001f\u0000\u029c\u029b\u0001\u0000\u0000\u0000\u029d"+
-    "\u029e\u0001\u0000\u0000\u0000\u029e\u029c\u0001\u0000\u0000\u0000\u029e"+
-    "\u029f\u0001\u0000\u0000\u0000\u029fc\u0001\u0000\u0000\u0000\u02a0\u02a2"+
-    "\u0003L\u001f\u0000\u02a1\u02a0\u0001\u0000\u0000\u0000\u02a2\u02a3\u0001"+
-    "\u0000\u0000\u0000\u02a3\u02a1\u0001\u0000\u0000\u0000\u02a3\u02a4\u0001"+
-    "\u0000\u0000\u0000\u02a4\u02a5\u0001\u0000\u0000\u0000\u02a5\u02a9\u0003"+
-    "t3\u0000\u02a6\u02a8\u0003L\u001f\u0000\u02a7\u02a6\u0001\u0000\u0000"+
-    "\u0000\u02a8\u02ab\u0001\u0000\u0000\u0000\u02a9\u02a7\u0001\u0000\u0000"+
-    "\u0000\u02a9\u02aa\u0001\u0000\u0000\u0000\u02aa\u02cb\u0001\u0000\u0000"+
-    "\u0000\u02ab\u02a9\u0001\u0000\u0000\u0000\u02ac\u02ae\u0003t3\u0000\u02ad"+
-    "\u02af\u0003L\u001f\u0000\u02ae\u02ad\u0001\u0000\u0000\u0000\u02af\u02b0"+
-    "\u0001\u0000\u0000\u0000\u02b0\u02ae\u0001\u0000\u0000\u0000\u02b0\u02b1"+
-    "\u0001\u0000\u0000\u0000\u02b1\u02cb\u0001\u0000\u0000\u0000\u02b2\u02b4"+
-    "\u0003L\u001f\u0000\u02b3\u02b2\u0001\u0000\u0000\u0000\u02b4\u02b5\u0001"+
-    "\u0000\u0000\u0000\u02b5\u02b3\u0001\u0000\u0000\u0000\u02b5\u02b6\u0001"+
-    "\u0000\u0000\u0000\u02b6\u02be\u0001\u0000\u0000\u0000\u02b7\u02bb\u0003"+
-    "t3\u0000\u02b8\u02ba\u0003L\u001f\u0000\u02b9\u02b8\u0001\u0000\u0000"+
-    "\u0000\u02ba\u02bd\u0001\u0000\u0000\u0000\u02bb\u02b9\u0001\u0000\u0000"+
-    "\u0000\u02bb\u02bc\u0001\u0000\u0000\u0000\u02bc\u02bf\u0001\u0000\u0000"+
-    "\u0000\u02bd\u02bb\u0001\u0000\u0000\u0000\u02be\u02b7\u0001\u0000\u0000"+
-    "\u0000\u02be\u02bf\u0001\u0000\u0000\u0000\u02bf\u02c0\u0001\u0000\u0000"+
-    "\u0000\u02c0\u02c1\u0003T#\u0000\u02c1\u02cb\u0001\u0000\u0000\u0000\u02c2"+
-    "\u02c4\u0003t3\u0000\u02c3\u02c5\u0003L\u001f\u0000\u02c4\u02c3\u0001"+
-    "\u0000\u0000\u0000\u02c5\u02c6\u0001\u0000\u0000\u0000\u02c6\u02c4\u0001"+
-    "\u0000\u0000\u0000\u02c6\u02c7\u0001\u0000\u0000\u0000\u02c7\u02c8\u0001"+
-    "\u0000\u0000\u0000\u02c8\u02c9\u0003T#\u0000\u02c9\u02cb\u0001\u0000\u0000"+
-    "\u0000\u02ca\u02a1\u0001\u0000\u0000\u0000\u02ca\u02ac\u0001\u0000\u0000"+
-    "\u0000\u02ca\u02b3\u0001\u0000\u0000\u0000\u02ca\u02c2\u0001\u0000\u0000"+
-    "\u0000\u02cbe\u0001\u0000\u0000\u0000\u02cc\u02cd\u0005b\u0000\u0000\u02cd"+
-    "\u02ce\u0005y\u0000\u0000\u02ceg\u0001\u0000\u0000\u0000\u02cf\u02d0\u0005"+
-    "a\u0000\u0000\u02d0\u02d1\u0005n\u0000\u0000\u02d1\u02d2\u0005d\u0000"+
-    "\u0000\u02d2i\u0001\u0000\u0000\u0000\u02d3\u02d4\u0005a\u0000\u0000\u02d4"+
-    "\u02d5\u0005s\u0000\u0000\u02d5\u02d6\u0005c\u0000\u0000\u02d6k\u0001"+
-    "\u0000\u0000\u0000\u02d7\u02d8\u0005=\u0000\u0000\u02d8m\u0001\u0000\u0000"+
-    "\u0000\u02d9\u02da\u0005:\u0000\u0000\u02da\u02db\u0005:\u0000\u0000\u02db"+
-    "o\u0001\u0000\u0000\u0000\u02dc\u02dd\u0005,\u0000\u0000\u02ddq\u0001"+
-    "\u0000\u0000\u0000\u02de\u02df\u0005d\u0000\u0000\u02df\u02e0\u0005e\u0000"+
-    "\u0000\u02e0\u02e1\u0005s\u0000\u0000\u02e1\u02e2\u0005c\u0000\u0000\u02e2"+
-    "s\u0001\u0000\u0000\u0000\u02e3\u02e4\u0005.\u0000\u0000\u02e4u\u0001"+
-    "\u0000\u0000\u0000\u02e5\u02e6\u0005f\u0000\u0000\u02e6\u02e7\u0005a\u0000"+
-    "\u0000\u02e7\u02e8\u0005l\u0000\u0000\u02e8\u02e9\u0005s\u0000\u0000\u02e9"+
-    "\u02ea\u0005e\u0000\u0000\u02eaw\u0001\u0000\u0000\u0000\u02eb\u02ec\u0005"+
-    "f\u0000\u0000\u02ec\u02ed\u0005i\u0000\u0000\u02ed\u02ee\u0005r\u0000"+
-    "\u0000\u02ee\u02ef\u0005s\u0000\u0000\u02ef\u02f0\u0005t\u0000\u0000\u02f0"+
-    "y\u0001\u0000\u0000\u0000\u02f1\u02f2\u0005l\u0000\u0000\u02f2\u02f3\u0005"+
-    "a\u0000\u0000\u02f3\u02f4\u0005s\u0000\u0000\u02f4\u02f5\u0005t\u0000"+
-    "\u0000\u02f5{\u0001\u0000\u0000\u0000\u02f6\u02f7\u0005(\u0000\u0000\u02f7"+
-    "}\u0001\u0000\u0000\u0000\u02f8\u02f9\u0005i\u0000\u0000\u02f9\u02fa\u0005"+
-    "n\u0000\u0000\u02fa\u007f\u0001\u0000\u0000\u0000\u02fb\u02fc\u0005i\u0000"+
-    "\u0000\u02fc\u02fd\u0005s\u0000\u0000\u02fd\u0081\u0001\u0000\u0000\u0000"+
-    "\u02fe\u02ff\u0005l\u0000\u0000\u02ff\u0300\u0005i\u0000\u0000\u0300\u0301"+
-    "\u0005k\u0000\u0000\u0301\u0302\u0005e\u0000\u0000\u0302\u0083\u0001\u0000"+
-    "\u0000\u0000\u0303\u0304\u0005n\u0000\u0000\u0304\u0305\u0005o\u0000\u0000"+
-    "\u0305\u0306\u0005t\u0000\u0000\u0306\u0085\u0001\u0000\u0000\u0000\u0307"+
-    "\u0308\u0005n\u0000\u0000\u0308\u0309\u0005u\u0000\u0000\u0309\u030a\u0005"+
-    "l\u0000\u0000\u030a\u030b\u0005l\u0000\u0000\u030b\u0087\u0001\u0000\u0000"+
-    "\u0000\u030c\u030d\u0005n\u0000\u0000\u030d\u030e\u0005u\u0000\u0000\u030e"+
-    "\u030f\u0005l\u0000\u0000\u030f\u0310\u0005l\u0000\u0000\u0310\u0311\u0005"+
-    "s\u0000\u0000\u0311\u0089\u0001\u0000\u0000\u0000\u0312\u0313\u0005o\u0000"+
-    "\u0000\u0313\u0314\u0005r\u0000\u0000\u0314\u008b\u0001\u0000\u0000\u0000"+
-    "\u0315\u0316\u0005?\u0000\u0000\u0316\u008d\u0001\u0000\u0000\u0000\u0317"+
-    "\u0318\u0005r\u0000\u0000\u0318\u0319\u0005l\u0000\u0000\u0319\u031a\u0005"+
-    "i\u0000\u0000\u031a\u031b\u0005k\u0000\u0000\u031b\u031c\u0005e\u0000"+
-    "\u0000\u031c\u008f\u0001\u0000\u0000\u0000\u031d\u031e\u0005)\u0000\u0000"+
-    "\u031e\u0091\u0001\u0000\u0000\u0000\u031f\u0320\u0005t\u0000\u0000\u0320"+
-    "\u0321\u0005r\u0000\u0000\u0321\u0322\u0005u\u0000\u0000\u0322\u0323\u0005"+
-    "e\u0000\u0000\u0323\u0093\u0001\u0000\u0000\u0000\u0324\u0325\u0005=\u0000"+
-    "\u0000\u0325\u0326\u0005=\u0000\u0000\u0326\u0095\u0001\u0000\u0000\u0000"+
-    "\u0327\u0328\u0005=\u0000\u0000\u0328\u0329\u0005~\u0000\u0000\u0329\u0097"+
-    "\u0001\u0000\u0000\u0000\u032a\u032b\u0005!\u0000\u0000\u032b\u032c\u0005"+
-    "=\u0000\u0000\u032c\u0099\u0001\u0000\u0000\u0000\u032d\u032e\u0005<\u0000"+
-    "\u0000\u032e\u009b\u0001\u0000\u0000\u0000\u032f\u0330\u0005<\u0000\u0000"+
-    "\u0330\u0331\u0005=\u0000\u0000\u0331\u009d\u0001\u0000\u0000\u0000\u0332"+
-    "\u0333\u0005>\u0000\u0000\u0333\u009f\u0001\u0000\u0000\u0000\u0334\u0335"+
-    "\u0005>\u0000\u0000\u0335\u0336\u0005=\u0000\u0000\u0336\u00a1\u0001\u0000"+
-    "\u0000\u0000\u0337\u0338\u0005+\u0000\u0000\u0338\u00a3\u0001\u0000\u0000"+
-    "\u0000\u0339\u033a\u0005-\u0000\u0000\u033a\u00a5\u0001\u0000\u0000\u0000"+
-    "\u033b\u033c\u0005*\u0000\u0000\u033c\u00a7\u0001\u0000\u0000\u0000\u033d"+
-    "\u033e\u0005/\u0000\u0000\u033e\u00a9\u0001\u0000\u0000\u0000\u033f\u0340"+
-    "\u0005%\u0000\u0000\u0340\u00ab\u0001\u0000\u0000\u0000\u0341\u0342\u0005"+
-    "[\u0000\u0000\u0342\u0343\u0001\u0000\u0000\u0000\u0343\u0344\u0006O\u0000"+
-    "\u0000\u0344\u0345\u0006O\u0000\u0000\u0345\u00ad\u0001\u0000\u0000\u0000"+
-    "\u0346\u0347\u0005]\u0000\u0000\u0347\u0348\u0001\u0000\u0000\u0000\u0348"+
-    "\u0349\u0006P\u000e\u0000\u0349\u034a\u0006P\u000e\u0000\u034a\u00af\u0001"+
-    "\u0000\u0000\u0000\u034b\u034f\u0003N \u0000\u034c\u034e\u0003^(\u0000"+
-    "\u034d\u034c\u0001\u0000\u0000\u0000\u034e\u0351\u0001\u0000\u0000\u0000"+
-    "\u034f\u034d\u0001\u0000\u0000\u0000\u034f\u0350\u0001\u0000\u0000\u0000"+
-    "\u0350\u035c\u0001\u0000\u0000\u0000\u0351\u034f\u0001\u0000\u0000\u0000"+
-    "\u0352\u0355\u0003\\\'\u0000\u0353\u0355\u0003V$\u0000\u0354\u0352\u0001"+
-    "\u0000\u0000\u0000\u0354\u0353\u0001\u0000\u0000\u0000\u0355\u0357\u0001"+
-    "\u0000\u0000\u0000\u0356\u0358\u0003^(\u0000\u0357\u0356\u0001\u0000\u0000"+
-    "\u0000\u0358\u0359\u0001\u0000\u0000\u0000\u0359\u0357\u0001\u0000\u0000"+
-    "\u0000\u0359\u035a\u0001\u0000\u0000\u0000\u035a\u035c\u0001\u0000\u0000"+
-    "\u0000\u035b\u034b\u0001\u0000\u0000\u0000\u035b\u0354\u0001\u0000\u0000"+
-    "\u0000\u035c\u00b1\u0001\u0000\u0000\u0000\u035d\u035f\u0003X%\u0000\u035e"+
-    "\u0360\u0003Z&\u0000\u035f\u035e\u0001\u0000\u0000\u0000\u0360\u0361\u0001"+
-    "\u0000\u0000\u0000\u0361\u035f\u0001\u0000\u0000\u0000\u0361\u0362\u0001"+
-    "\u0000\u0000\u0000\u0362\u0363\u0001\u0000\u0000\u0000\u0363\u0364\u0003"+
-    "X%\u0000\u0364\u00b3\u0001\u0000\u0000\u0000\u0365\u0366\u0003\u00b2R"+
-    "\u0000\u0366\u00b5\u0001\u0000\u0000\u0000\u0367\u0368\u00036\u0014\u0000"+
-    "\u0368\u0369\u0001\u0000\u0000\u0000\u0369\u036a\u0006T\n\u0000\u036a"+
-    "\u00b7\u0001\u0000\u0000\u0000\u036b\u036c\u00038\u0015\u0000\u036c\u036d"+
-    "\u0001\u0000\u0000\u0000\u036d\u036e\u0006U\n\u0000\u036e\u00b9\u0001"+
-    "\u0000\u0000\u0000\u036f\u0370\u0003:\u0016\u0000\u0370\u0371\u0001\u0000"+
-    "\u0000\u0000\u0371\u0372\u0006V\n\u0000\u0372\u00bb\u0001\u0000\u0000"+
-    "\u0000\u0373\u0374\u0003J\u001e\u0000\u0374\u0375\u0001\u0000\u0000\u0000"+
-    "\u0375\u0376\u0006W\r\u0000\u0376\u0377\u0006W\u000e\u0000\u0377\u00bd"+
-    "\u0001\u0000\u0000\u0000\u0378\u0379\u0003\u00acO\u0000\u0379\u037a\u0001"+
-    "\u0000\u0000\u0000\u037a\u037b\u0006X\u000b\u0000\u037b\u00bf\u0001\u0000"+
-    "\u0000\u0000\u037c\u037d\u0003\u00aeP\u0000\u037d\u037e\u0001\u0000\u0000"+
-    "\u0000\u037e\u037f\u0006Y\u000f\u0000\u037f\u00c1\u0001\u0000\u0000\u0000"+
-    "\u0380\u0381\u0003p1\u0000\u0381\u0382\u0001\u0000\u0000\u0000\u0382\u0383"+
-    "\u0006Z\u0010\u0000\u0383\u00c3\u0001\u0000\u0000\u0000\u0384\u0385\u0003"+
-    "l/\u0000\u0385\u0386\u0001\u0000\u0000\u0000\u0386\u0387\u0006[\u0011"+
-    "\u0000\u0387\u00c5\u0001\u0000\u0000\u0000\u0388\u0389\u0003`)\u0000\u0389"+
-    "\u038a\u0001\u0000\u0000\u0000\u038a\u038b\u0006\\\u0012\u0000\u038b\u00c7"+
-    "\u0001\u0000\u0000\u0000\u038c\u038d\u0005m\u0000\u0000\u038d\u038e\u0005"+
-    "e\u0000\u0000\u038e\u038f\u0005t\u0000\u0000\u038f\u0390\u0005a\u0000"+
-    "\u0000\u0390\u0391\u0005d\u0000\u0000\u0391\u0392\u0005a\u0000\u0000\u0392"+
-    "\u0393\u0005t\u0000\u0000\u0393\u0394\u0005a\u0000\u0000\u0394\u00c9\u0001"+
-    "\u0000\u0000\u0000\u0395\u0396\u0003>\u0018\u0000\u0396\u0397\u0001\u0000"+
-    "\u0000\u0000\u0397\u0398\u0006^\u0013\u0000\u0398\u00cb\u0001\u0000\u0000"+
-    "\u0000\u0399\u039a\u00036\u0014\u0000\u039a\u039b\u0001\u0000\u0000\u0000"+
-    "\u039b\u039c\u0006_\n\u0000\u039c\u00cd\u0001\u0000\u0000\u0000\u039d"+
-    "\u039e\u00038\u0015\u0000\u039e\u039f\u0001\u0000\u0000\u0000\u039f\u03a0"+
-    "\u0006`\n\u0000\u03a0\u00cf\u0001\u0000\u0000\u0000\u03a1\u03a2\u0003"+
-    ":\u0016\u0000\u03a2\u03a3\u0001\u0000\u0000\u0000\u03a3\u03a4\u0006a\n"+
-    "\u0000\u03a4\u00d1\u0001\u0000\u0000\u0000\u03a5\u03a6\u0003J\u001e\u0000"+
-    "\u03a6\u03a7\u0001\u0000\u0000\u0000\u03a7\u03a8\u0006b\r\u0000\u03a8"+
-    "\u03a9\u0006b\u000e\u0000\u03a9\u00d3\u0001\u0000\u0000\u0000\u03aa\u03ab"+
-    "\u0003t3\u0000\u03ab\u03ac\u0001\u0000\u0000\u0000\u03ac\u03ad\u0006c"+
-    "\u0014\u0000\u03ad\u00d5\u0001\u0000\u0000\u0000\u03ae\u03af\u0003p1\u0000"+
-    "\u03af\u03b0\u0001\u0000\u0000\u0000\u03b0\u03b1\u0006d\u0010\u0000\u03b1"+
-    "\u00d7\u0001\u0000\u0000\u0000\u03b2\u03b7\u0003N \u0000\u03b3\u03b7\u0003"+
-    "L\u001f\u0000\u03b4\u03b7\u0003\\\'\u0000\u03b5\u03b7\u0003\u00a6L\u0000"+
-    "\u03b6\u03b2\u0001\u0000\u0000\u0000\u03b6\u03b3\u0001\u0000\u0000\u0000"+
-    "\u03b6\u03b4\u0001\u0000\u0000\u0000\u03b6\u03b5\u0001\u0000\u0000\u0000"+
-    "\u03b7\u00d9\u0001\u0000\u0000\u0000\u03b8\u03bb\u0003N \u0000\u03b9\u03bb"+
-    "\u0003\u00a6L\u0000\u03ba\u03b8\u0001\u0000\u0000\u0000\u03ba\u03b9\u0001"+
-    "\u0000\u0000\u0000\u03bb\u03bf\u0001\u0000\u0000\u0000\u03bc\u03be\u0003"+
-    "\u00d8e\u0000\u03bd\u03bc\u0001\u0000\u0000\u0000\u03be\u03c1\u0001\u0000"+
-    "\u0000\u0000\u03bf\u03bd\u0001\u0000\u0000\u0000\u03bf\u03c0\u0001\u0000"+
-    "\u0000\u0000\u03c0\u03cc\u0001\u0000\u0000\u0000\u03c1\u03bf\u0001\u0000"+
-    "\u0000\u0000\u03c2\u03c5\u0003\\\'\u0000\u03c3\u03c5\u0003V$\u0000\u03c4"+
-    "\u03c2\u0001\u0000\u0000\u0000\u03c4\u03c3\u0001\u0000\u0000\u0000\u03c5"+
-    "\u03c7\u0001\u0000\u0000\u0000\u03c6\u03c8\u0003\u00d8e\u0000\u03c7\u03c6"+
-    "\u0001\u0000\u0000\u0000\u03c8\u03c9\u0001\u0000\u0000\u0000\u03c9\u03c7"+
-    "\u0001\u0000\u0000\u0000\u03c9\u03ca\u0001\u0000\u0000\u0000\u03ca\u03cc"+
-    "\u0001\u0000\u0000\u0000\u03cb\u03ba\u0001\u0000\u0000\u0000\u03cb\u03c4"+
-    "\u0001\u0000\u0000\u0000\u03cc\u00db\u0001\u0000\u0000\u0000\u03cd\u03d0"+
-    "\u0003\u00daf\u0000\u03ce\u03d0\u0003\u00b2R\u0000\u03cf\u03cd\u0001\u0000"+
-    "\u0000\u0000\u03cf\u03ce\u0001\u0000\u0000\u0000\u03d0\u03d1\u0001\u0000"+
-    "\u0000\u0000\u03d1\u03cf\u0001\u0000\u0000\u0000\u03d1\u03d2\u0001\u0000"+
-    "\u0000\u0000\u03d2\u00dd\u0001\u0000\u0000\u0000\u03d3\u03d4\u00036\u0014"+
-    "\u0000\u03d4\u03d5\u0001\u0000\u0000\u0000\u03d5\u03d6\u0006h\n\u0000"+
-    "\u03d6\u00df\u0001\u0000\u0000\u0000\u03d7\u03d8\u00038\u0015\u0000\u03d8"+
-    "\u03d9\u0001\u0000\u0000\u0000\u03d9\u03da\u0006i\n\u0000\u03da\u00e1"+
-    "\u0001\u0000\u0000\u0000\u03db\u03dc\u0003:\u0016\u0000\u03dc\u03dd\u0001"+
-    "\u0000\u0000\u0000\u03dd\u03de\u0006j\n\u0000\u03de\u00e3\u0001\u0000"+
-    "\u0000\u0000\u03df\u03e0\u0003J\u001e\u0000\u03e0\u03e1\u0001\u0000\u0000"+
-    "\u0000\u03e1\u03e2\u0006k\r\u0000\u03e2\u03e3\u0006k\u000e\u0000\u03e3"+
-    "\u00e5\u0001\u0000\u0000\u0000\u03e4\u03e5\u0003l/\u0000\u03e5\u03e6\u0001"+
-    "\u0000\u0000\u0000\u03e6\u03e7\u0006l\u0011\u0000\u03e7\u00e7\u0001\u0000"+
-    "\u0000\u0000\u03e8\u03e9\u0003p1\u0000\u03e9\u03ea\u0001\u0000\u0000\u0000"+
-    "\u03ea\u03eb\u0006m\u0010\u0000\u03eb\u00e9\u0001\u0000\u0000\u0000\u03ec"+
-    "\u03ed\u0003t3\u0000\u03ed\u03ee\u0001\u0000\u0000\u0000\u03ee\u03ef\u0006"+
-    "n\u0014\u0000\u03ef\u00eb\u0001\u0000\u0000\u0000\u03f0\u03f1\u0005a\u0000"+
-    "\u0000\u03f1\u03f2\u0005s\u0000\u0000\u03f2\u00ed\u0001\u0000\u0000\u0000"+
-    "\u03f3\u03f4\u0003\u00dcg\u0000\u03f4\u03f5\u0001\u0000\u0000\u0000\u03f5"+
-    "\u03f6\u0006p\u0015\u0000\u03f6\u00ef\u0001\u0000\u0000\u0000\u03f7\u03f8"+
-    "\u00036\u0014\u0000\u03f8\u03f9\u0001\u0000\u0000\u0000\u03f9\u03fa\u0006"+
-    "q\n\u0000\u03fa\u00f1\u0001\u0000\u0000\u0000\u03fb\u03fc\u00038\u0015"+
-    "\u0000\u03fc\u03fd\u0001\u0000\u0000\u0000\u03fd\u03fe\u0006r\n\u0000"+
-    "\u03fe\u00f3\u0001\u0000\u0000\u0000\u03ff\u0400\u0003:\u0016\u0000\u0400"+
-    "\u0401\u0001\u0000\u0000\u0000\u0401\u0402\u0006s\n\u0000\u0402\u00f5"+
-    "\u0001\u0000\u0000\u0000\u0403\u0404\u0003J\u001e\u0000\u0404\u0405\u0001"+
-    "\u0000\u0000\u0000\u0405\u0406\u0006t\r\u0000\u0406\u0407\u0006t\u000e"+
-    "\u0000\u0407\u00f7\u0001\u0000\u0000\u0000\u0408\u0409\u0003\u00acO\u0000"+
-    "\u0409\u040a\u0001\u0000\u0000\u0000\u040a\u040b\u0006u\u000b\u0000\u040b"+
-    "\u040c\u0006u\u0016\u0000\u040c\u00f9\u0001\u0000\u0000\u0000\u040d\u040e"+
-    "\u0005o\u0000\u0000\u040e\u040f\u0005n\u0000\u0000\u040f\u0410\u0001\u0000"+
-    "\u0000\u0000\u0410\u0411\u0006v\u0017\u0000\u0411\u00fb\u0001\u0000\u0000"+
-    "\u0000\u0412\u0413\u0005w\u0000\u0000\u0413\u0414\u0005i\u0000\u0000\u0414"+
-    "\u0415\u0005t\u0000\u0000\u0415\u0416\u0005h\u0000\u0000\u0416\u0417\u0001"+
-    "\u0000\u0000\u0000\u0417\u0418\u0006w\u0017\u0000\u0418\u00fd\u0001\u0000"+
-    "\u0000\u0000\u0419\u041a\b\f\u0000\u0000\u041a\u00ff\u0001\u0000\u0000"+
-    "\u0000\u041b\u041d\u0003\u00fex\u0000\u041c\u041b\u0001\u0000\u0000\u0000"+
-    "\u041d\u041e\u0001\u0000\u0000\u0000\u041e\u041c\u0001\u0000\u0000\u0000"+
-    "\u041e\u041f\u0001\u0000\u0000\u0000\u041f\u0420\u0001\u0000\u0000\u0000"+
-    "\u0420\u0421\u0003\u0144\u009b\u0000\u0421\u0423\u0001\u0000\u0000\u0000"+
-    "\u0422\u041c\u0001\u0000\u0000\u0000\u0422\u0423\u0001\u0000\u0000\u0000"+
-    "\u0423\u0425\u0001\u0000\u0000\u0000\u0424\u0426\u0003\u00fex\u0000\u0425"+
-    "\u0424\u0001\u0000\u0000\u0000\u0426\u0427\u0001\u0000\u0000\u0000\u0427"+
-    "\u0425\u0001\u0000\u0000\u0000\u0427\u0428\u0001\u0000\u0000\u0000\u0428"+
-    "\u0101\u0001\u0000\u0000\u0000\u0429\u042a\u0003\u00b4S\u0000\u042a\u042b"+
-    "\u0001\u0000\u0000\u0000\u042b\u042c\u0006z\u0018\u0000\u042c\u0103\u0001"+
-    "\u0000\u0000\u0000\u042d\u042e\u0003\u0100y\u0000\u042e\u042f\u0001\u0000"+
-    "\u0000\u0000\u042f\u0430\u0006{\u0019\u0000\u0430\u0105\u0001\u0000\u0000"+
-    "\u0000\u0431\u0432\u00036\u0014\u0000\u0432\u0433\u0001\u0000\u0000\u0000"+
-    "\u0433\u0434\u0006|\n\u0000\u0434\u0107\u0001\u0000\u0000\u0000\u0435"+
-    "\u0436\u00038\u0015\u0000\u0436\u0437\u0001\u0000\u0000\u0000\u0437\u0438"+
-    "\u0006}\n\u0000\u0438\u0109\u0001\u0000\u0000\u0000\u0439\u043a\u0003"+
-    ":\u0016\u0000\u043a\u043b\u0001\u0000\u0000\u0000\u043b\u043c\u0006~\n"+
-    "\u0000\u043c\u010b\u0001\u0000\u0000\u0000\u043d\u043e\u0003J\u001e\u0000"+
-    "\u043e\u043f\u0001\u0000\u0000\u0000\u043f\u0440\u0006\u007f\r\u0000\u0440"+
-    "\u0441\u0006\u007f\u000e\u0000\u0441\u0442\u0006\u007f\u000e\u0000\u0442"+
-    "\u010d\u0001\u0000\u0000\u0000\u0443\u0444\u0003l/\u0000\u0444\u0445\u0001"+
-    "\u0000\u0000\u0000\u0445\u0446\u0006\u0080\u0011\u0000\u0446\u010f\u0001"+
-    "\u0000\u0000\u0000\u0447\u0448\u0003p1\u0000\u0448\u0449\u0001\u0000\u0000"+
-    "\u0000\u0449\u044a\u0006\u0081\u0010\u0000\u044a\u0111\u0001\u0000\u0000"+
-    "\u0000\u044b\u044c\u0003t3\u0000\u044c\u044d\u0001\u0000\u0000\u0000\u044d"+
-    "\u044e\u0006\u0082\u0014\u0000\u044e\u0113\u0001\u0000\u0000\u0000\u044f"+
-    "\u0450\u0003\u00fcw\u0000\u0450\u0451\u0001\u0000\u0000\u0000\u0451\u0452"+
-    "\u0006\u0083\u001a\u0000\u0452\u0115\u0001\u0000\u0000\u0000\u0453\u0454"+
-    "\u0003\u00dcg\u0000\u0454\u0455\u0001\u0000\u0000\u0000\u0455\u0456\u0006"+
-    "\u0084\u0015\u0000\u0456\u0117\u0001\u0000\u0000\u0000\u0457\u0458\u0003"+
-    "\u00b4S\u0000\u0458\u0459\u0001\u0000\u0000\u0000\u0459\u045a\u0006\u0085"+
-    "\u0018\u0000\u045a\u0119\u0001\u0000\u0000\u0000\u045b\u045c\u00036\u0014"+
-    "\u0000\u045c\u045d\u0001\u0000\u0000\u0000\u045d\u045e\u0006\u0086\n\u0000"+
-    "\u045e\u011b\u0001\u0000\u0000\u0000\u045f\u0460\u00038\u0015\u0000\u0460"+
-    "\u0461\u0001\u0000\u0000\u0000\u0461\u0462\u0006\u0087\n\u0000\u0462\u011d"+
-    "\u0001\u0000\u0000\u0000\u0463\u0464\u0003:\u0016\u0000\u0464\u0465\u0001"+
-    "\u0000\u0000\u0000\u0465\u0466\u0006\u0088\n\u0000\u0466\u011f\u0001\u0000"+
-    "\u0000\u0000\u0467\u0468\u0003J\u001e\u0000\u0468\u0469\u0001\u0000\u0000"+
-    "\u0000\u0469\u046a\u0006\u0089\r\u0000\u046a\u046b\u0006\u0089\u000e\u0000"+
-    "\u046b\u0121\u0001\u0000\u0000\u0000\u046c\u046d\u0003t3\u0000\u046d\u046e"+
-    "\u0001\u0000\u0000\u0000\u046e\u046f\u0006\u008a\u0014\u0000\u046f\u0123"+
-    "\u0001\u0000\u0000\u0000\u0470\u0471\u0003\u00b4S\u0000\u0471\u0472\u0001"+
-    "\u0000\u0000\u0000\u0472\u0473\u0006\u008b\u0018\u0000\u0473\u0125\u0001"+
-    "\u0000\u0000\u0000\u0474\u0475\u0003\u00b0Q\u0000\u0475\u0476\u0001\u0000"+
-    "\u0000\u0000\u0476\u0477\u0006\u008c\u001b\u0000\u0477\u0127\u0001\u0000"+
-    "\u0000\u0000\u0478\u0479\u00036\u0014\u0000\u0479\u047a\u0001\u0000\u0000"+
-    "\u0000\u047a\u047b\u0006\u008d\n\u0000\u047b\u0129\u0001\u0000\u0000\u0000"+
-    "\u047c\u047d\u00038\u0015\u0000\u047d\u047e\u0001\u0000\u0000\u0000\u047e"+
-    "\u047f\u0006\u008e\n\u0000\u047f\u012b\u0001\u0000\u0000\u0000\u0480\u0481"+
-    "\u0003:\u0016\u0000\u0481\u0482\u0001\u0000\u0000\u0000\u0482\u0483\u0006"+
-    "\u008f\n\u0000\u0483\u012d\u0001\u0000\u0000\u0000\u0484\u0485\u0003J"+
-    "\u001e\u0000\u0485\u0486\u0001\u0000\u0000\u0000\u0486\u0487\u0006\u0090"+
-    "\r\u0000\u0487\u0488\u0006\u0090\u000e\u0000\u0488\u012f\u0001\u0000\u0000"+
-    "\u0000\u0489\u048a\u0005i\u0000\u0000\u048a\u048b\u0005n\u0000\u0000\u048b"+
-    "\u048c\u0005f\u0000\u0000\u048c\u048d\u0005o\u0000\u0000\u048d\u0131\u0001"+
-    "\u0000\u0000\u0000\u048e\u048f\u00036\u0014\u0000\u048f\u0490\u0001\u0000"+
-    "\u0000\u0000\u0490\u0491\u0006\u0092\n\u0000\u0491\u0133\u0001\u0000\u0000"+
-    "\u0000\u0492\u0493\u00038\u0015\u0000\u0493\u0494\u0001\u0000\u0000\u0000"+
-    "\u0494\u0495\u0006\u0093\n\u0000\u0495\u0135\u0001\u0000\u0000\u0000\u0496"+
-    "\u0497\u0003:\u0016\u0000\u0497\u0498\u0001\u0000\u0000\u0000\u0498\u0499"+
-    "\u0006\u0094\n\u0000\u0499\u0137\u0001\u0000\u0000\u0000\u049a\u049b\u0003"+
-    "J\u001e\u0000\u049b\u049c\u0001\u0000\u0000\u0000\u049c\u049d\u0006\u0095"+
-    "\r\u0000\u049d\u049e\u0006\u0095\u000e\u0000\u049e\u0139\u0001\u0000\u0000"+
-    "\u0000\u049f\u04a0\u0005f\u0000\u0000\u04a0\u04a1\u0005u\u0000\u0000\u04a1"+
-    "\u04a2\u0005n\u0000\u0000\u04a2\u04a3\u0005c\u0000\u0000\u04a3\u04a4\u0005"+
-    "t\u0000\u0000\u04a4\u04a5\u0005i\u0000\u0000\u04a5\u04a6\u0005o\u0000"+
-    "\u0000\u04a6\u04a7\u0005n\u0000\u0000\u04a7\u04a8\u0005s\u0000\u0000\u04a8"+
-    "\u013b\u0001\u0000\u0000\u0000\u04a9\u04aa\u00036\u0014\u0000\u04aa\u04ab"+
-    "\u0001\u0000\u0000\u0000\u04ab\u04ac\u0006\u0097\n\u0000\u04ac\u013d\u0001"+
-    "\u0000\u0000\u0000\u04ad\u04ae\u00038\u0015\u0000\u04ae\u04af\u0001\u0000"+
-    "\u0000\u0000\u04af\u04b0\u0006\u0098\n\u0000\u04b0\u013f\u0001\u0000\u0000"+
-    "\u0000\u04b1\u04b2\u0003:\u0016\u0000\u04b2\u04b3\u0001\u0000\u0000\u0000"+
-    "\u04b3\u04b4\u0006\u0099\n\u0000\u04b4\u0141\u0001\u0000\u0000\u0000\u04b5"+
-    "\u04b6\u0003\u00aeP\u0000\u04b6\u04b7\u0001\u0000\u0000\u0000\u04b7\u04b8"+
-    "\u0006\u009a\u000f\u0000\u04b8\u04b9\u0006\u009a\u000e\u0000\u04b9\u0143"+
-    "\u0001\u0000\u0000\u0000\u04ba\u04bb\u0005:\u0000\u0000\u04bb\u0145\u0001"+
-    "\u0000\u0000\u0000\u04bc\u04c2\u0003V$\u0000\u04bd\u04c2\u0003L\u001f"+
-    "\u0000\u04be\u04c2\u0003t3\u0000\u04bf\u04c2\u0003N \u0000\u04c0\u04c2"+
-    "\u0003\\\'\u0000\u04c1\u04bc\u0001\u0000\u0000\u0000\u04c1\u04bd\u0001"+
-    "\u0000\u0000\u0000\u04c1\u04be\u0001\u0000\u0000\u0000\u04c1\u04bf\u0001"+
-    "\u0000\u0000\u0000\u04c1\u04c0\u0001\u0000\u0000\u0000\u04c2\u04c3\u0001"+
-    "\u0000\u0000\u0000\u04c3\u04c1\u0001\u0000\u0000\u0000\u04c3\u04c4\u0001"+
-    "\u0000\u0000\u0000\u04c4\u0147\u0001\u0000\u0000\u0000\u04c5\u04c6\u0003"+
-    "6\u0014\u0000\u04c6\u04c7\u0001\u0000\u0000\u0000\u04c7\u04c8\u0006\u009d"+
-    "\n\u0000\u04c8\u0149\u0001\u0000\u0000\u0000\u04c9\u04ca\u00038\u0015"+
-    "\u0000\u04ca\u04cb\u0001\u0000\u0000\u0000\u04cb\u04cc\u0006\u009e\n\u0000"+
-    "\u04cc\u014b\u0001\u0000\u0000\u0000\u04cd\u04ce\u0003:\u0016\u0000\u04ce"+
-    "\u04cf\u0001\u0000\u0000\u0000\u04cf\u04d0\u0006\u009f\n\u0000\u04d0\u014d"+
-    "\u0001\u0000\u0000\u0000\u04d1\u04d2\u0003J\u001e\u0000\u04d2\u04d3\u0001"+
-    "\u0000\u0000\u0000\u04d3\u04d4\u0006\u00a0\r\u0000\u04d4\u04d5\u0006\u00a0"+
-    "\u000e\u0000\u04d5\u014f\u0001\u0000\u0000\u0000\u04d6\u04d7\u0003>\u0018"+
-    "\u0000\u04d7\u04d8\u0001\u0000\u0000\u0000\u04d8\u04d9\u0006\u00a1\u0013"+
-    "\u0000\u04d9\u04da\u0006\u00a1\u000e\u0000\u04da\u04db\u0006\u00a1\u001c"+
-    "\u0000\u04db\u0151\u0001\u0000\u0000\u0000\u04dc\u04dd\u00036\u0014\u0000"+
-    "\u04dd\u04de\u0001\u0000\u0000\u0000\u04de\u04df\u0006\u00a2\n\u0000\u04df"+
-    "\u0153\u0001\u0000\u0000\u0000\u04e0\u04e1\u00038\u0015\u0000\u04e1\u04e2"+
-    "\u0001\u0000\u0000\u0000\u04e2\u04e3\u0006\u00a3\n\u0000\u04e3\u0155\u0001"+
-    "\u0000\u0000\u0000\u04e4\u04e5\u0003:\u0016\u0000\u04e5\u04e6\u0001\u0000"+
-    "\u0000\u0000\u04e6\u04e7\u0006\u00a4\n\u0000\u04e7\u0157\u0001\u0000\u0000"+
-    "\u0000\u04e8\u04e9\u0003p1\u0000\u04e9\u04ea\u0001\u0000\u0000\u0000\u04ea"+
-    "\u04eb\u0006\u00a5\u0010\u0000\u04eb\u04ec\u0006\u00a5\u000e\u0000\u04ec"+
-    "\u04ed\u0006\u00a5\u0006\u0000\u04ed\u0159\u0001\u0000\u0000\u0000\u04ee"+
-    "\u04ef\u00036\u0014\u0000\u04ef\u04f0\u0001\u0000\u0000\u0000\u04f0\u04f1"+
-    "\u0006\u00a6\n\u0000\u04f1\u015b\u0001\u0000\u0000\u0000\u04f2\u04f3\u0003"+
-    "8\u0015\u0000\u04f3\u04f4\u0001\u0000\u0000\u0000\u04f4\u04f5\u0006\u00a7"+
-    "\n\u0000\u04f5\u015d\u0001\u0000\u0000\u0000\u04f6\u04f7\u0003:\u0016"+
-    "\u0000\u04f7\u04f8\u0001\u0000\u0000\u0000\u04f8\u04f9\u0006\u00a8\n\u0000"+
-    "\u04f9\u015f\u0001\u0000\u0000\u0000\u04fa\u04fb\u0003\u00b4S\u0000\u04fb"+
-    "\u04fc\u0001\u0000\u0000\u0000\u04fc\u04fd\u0006\u00a9\u000e\u0000\u04fd"+
-    "\u04fe\u0006\u00a9\u0000\u0000\u04fe\u04ff\u0006\u00a9\u0018\u0000\u04ff"+
-    "\u0161\u0001\u0000\u0000\u0000\u0500\u0501\u0003\u00b0Q\u0000\u0501\u0502"+
-    "\u0001\u0000\u0000\u0000\u0502\u0503\u0006\u00aa\u000e\u0000\u0503\u0504"+
-    "\u0006\u00aa\u0000\u0000\u0504\u0505\u0006\u00aa\u001b\u0000\u0505\u0163"+
-    "\u0001\u0000\u0000\u0000\u0506\u0507\u0003f,\u0000\u0507\u0508\u0001\u0000"+
-    "\u0000\u0000\u0508\u0509\u0006\u00ab\u000e\u0000\u0509\u050a\u0006\u00ab"+
-    "\u0000\u0000\u050a\u050b\u0006\u00ab\u001d\u0000\u050b\u0165\u0001\u0000"+
-    "\u0000\u0000\u050c\u050d\u0003J\u001e\u0000\u050d\u050e\u0001\u0000\u0000"+
-    "\u0000\u050e\u050f\u0006\u00ac\r\u0000\u050f\u0510\u0006\u00ac\u000e\u0000"+
-    "\u0510\u0167\u0001\u0000\u0000\u0000<\u0000\u0001\u0002\u0003\u0004\u0005"+
-    "\u0006\u0007\b\t\n\u000b\f\r\u020b\u0215\u0219\u021c\u0225\u0227\u0232"+
-    "\u0239\u023e\u0265\u026a\u0273\u027a\u027f\u0281\u028c\u0294\u0297\u0299"+
-    "\u029e\u02a3\u02a9\u02b0\u02b5\u02bb\u02be\u02c6\u02ca\u034f\u0354\u0359"+
-    "\u035b\u0361\u03b6\u03ba\u03bf\u03c4\u03c9\u03cb\u03cf\u03d1\u041e\u0422"+
-    "\u0427\u04c1\u04c3\u001e\u0005\u0002\u0000\u0005\u0004\u0000\u0005\u0006"+
-    "\u0000\u0005\u0001\u0000\u0005\u0003\u0000\u0005\n\u0000\u0005\f\u0000"+
-    "\u0005\b\u0000\u0005\u0005\u0000\u0005\t\u0000\u0000\u0001\u0000\u0007"+
-    "C\u0000\u0005\u0000\u0000\u0007\u001c\u0000\u0004\u0000\u0000\u0007D\u0000"+
-    "\u0007%\u0000\u0007#\u0000\u0007\u001d\u0000\u0007\u0018\u0000\u0007\'"+
-    "\u0000\u0007N\u0000\u0005\u000b\u0000\u0005\u0007\u0000\u0007F\u0000\u0007"+
-    "X\u0000\u0007W\u0000\u0007E\u0000\u0005\r\u0000\u0007 \u0000";
+    "\r\u0168\u0001\u0000\u0000\u0000\r\u016a\u0001\u0000\u0000\u0000\r\u016c"+
+    "\u0001\u0000\u0000\u0000\r\u016e\u0001\u0000\u0000\u0000\u000e\u0170\u0001"+
+    "\u0000\u0000\u0000\u000e\u0172\u0001\u0000\u0000\u0000\u000e\u0174\u0001"+
+    "\u0000\u0000\u0000\u000e\u0176\u0001\u0000\u0000\u0000\u000e\u0178\u0001"+
+    "\u0000\u0000\u0000\u000f\u017a\u0001\u0000\u0000\u0000\u000f\u017c\u0001"+
+    "\u0000\u0000\u0000\u000f\u017e\u0001\u0000\u0000\u0000\u000f\u0180\u0001"+
+    "\u0000\u0000\u0000\u000f\u0182\u0001\u0000\u0000\u0000\u000f\u0184\u0001"+
+    "\u0000\u0000\u0000\u000f\u0186\u0001\u0000\u0000\u0000\u000f\u0188\u0001"+
+    "\u0000\u0000\u0000\u0010\u018a\u0001\u0000\u0000\u0000\u0012\u0194\u0001"+
+    "\u0000\u0000\u0000\u0014\u019b\u0001\u0000\u0000\u0000\u0016\u01a4\u0001"+
+    "\u0000\u0000\u0000\u0018\u01ab\u0001\u0000\u0000\u0000\u001a\u01b5\u0001"+
+    "\u0000\u0000\u0000\u001c\u01bc\u0001\u0000\u0000\u0000\u001e\u01c3\u0001"+
+    "\u0000\u0000\u0000 \u01d1\u0001\u0000\u0000\u0000\"\u01d8\u0001\u0000"+
+    "\u0000\u0000$\u01e0\u0001\u0000\u0000\u0000&\u01e9\u0001\u0000\u0000\u0000"+
+    "(\u01f0\u0001\u0000\u0000\u0000*\u01fa\u0001\u0000\u0000\u0000,\u0206"+
+    "\u0001\u0000\u0000\u0000.\u020f\u0001\u0000\u0000\u00000\u0215\u0001\u0000"+
+    "\u0000\u00002\u021c\u0001\u0000\u0000\u00004\u0223\u0001\u0000\u0000\u0000"+
+    "6\u022b\u0001\u0000\u0000\u00008\u0234\u0001\u0000\u0000\u0000:\u023a"+
+    "\u0001\u0000\u0000\u0000<\u024b\u0001\u0000\u0000\u0000>\u025b\u0001\u0000"+
+    "\u0000\u0000@\u0264\u0001\u0000\u0000\u0000B\u0267\u0001\u0000\u0000\u0000"+
+    "D\u026b\u0001\u0000\u0000\u0000F\u0270\u0001\u0000\u0000\u0000H\u0275"+
+    "\u0001\u0000\u0000\u0000J\u0279\u0001\u0000\u0000\u0000L\u027d\u0001\u0000"+
+    "\u0000\u0000N\u0281\u0001\u0000\u0000\u0000P\u0285\u0001\u0000\u0000\u0000"+
+    "R\u0287\u0001\u0000\u0000\u0000T\u0289\u0001\u0000\u0000\u0000V\u028c"+
+    "\u0001\u0000\u0000\u0000X\u028e\u0001\u0000\u0000\u0000Z\u0297\u0001\u0000"+
+    "\u0000\u0000\\\u0299\u0001\u0000\u0000\u0000^\u029e\u0001\u0000\u0000"+
+    "\u0000`\u02a0\u0001\u0000\u0000\u0000b\u02a5\u0001\u0000\u0000\u0000d"+
+    "\u02c4\u0001\u0000\u0000\u0000f\u02c7\u0001\u0000\u0000\u0000h\u02f5\u0001"+
+    "\u0000\u0000\u0000j\u02f7\u0001\u0000\u0000\u0000l\u02fa\u0001\u0000\u0000"+
+    "\u0000n\u02fe\u0001\u0000\u0000\u0000p\u0302\u0001\u0000\u0000\u0000r"+
+    "\u0304\u0001\u0000\u0000\u0000t\u0307\u0001\u0000\u0000\u0000v\u0309\u0001"+
+    "\u0000\u0000\u0000x\u030e\u0001\u0000\u0000\u0000z\u0310\u0001\u0000\u0000"+
+    "\u0000|\u0316\u0001\u0000\u0000\u0000~\u031c\u0001\u0000\u0000\u0000\u0080"+
+    "\u0321\u0001\u0000\u0000\u0000\u0082\u0323\u0001\u0000\u0000\u0000\u0084"+
+    "\u0326\u0001\u0000\u0000\u0000\u0086\u0329\u0001\u0000\u0000\u0000\u0088"+
+    "\u032e\u0001\u0000\u0000\u0000\u008a\u0332\u0001\u0000\u0000\u0000\u008c"+
+    "\u0337\u0001\u0000\u0000\u0000\u008e\u033d\u0001\u0000\u0000\u0000\u0090"+
+    "\u0340\u0001\u0000\u0000\u0000\u0092\u0342\u0001\u0000\u0000\u0000\u0094"+
+    "\u0348\u0001\u0000\u0000\u0000\u0096\u034a\u0001\u0000\u0000\u0000\u0098"+
+    "\u034f\u0001\u0000\u0000\u0000\u009a\u0352\u0001\u0000\u0000\u0000\u009c"+
+    "\u0355\u0001\u0000\u0000\u0000\u009e\u0358\u0001\u0000\u0000\u0000\u00a0"+
+    "\u035a\u0001\u0000\u0000\u0000\u00a2\u035d\u0001\u0000\u0000\u0000\u00a4"+
+    "\u035f\u0001\u0000\u0000\u0000\u00a6\u0362\u0001\u0000\u0000\u0000\u00a8"+
+    "\u0364\u0001\u0000\u0000\u0000\u00aa\u0366\u0001\u0000\u0000\u0000\u00ac"+
+    "\u0368\u0001\u0000\u0000\u0000\u00ae\u036a\u0001\u0000\u0000\u0000\u00b0"+
+    "\u036c\u0001\u0000\u0000\u0000\u00b2\u0371\u0001\u0000\u0000\u0000\u00b4"+
+    "\u0386\u0001\u0000\u0000\u0000\u00b6\u0388\u0001\u0000\u0000\u0000\u00b8"+
+    "\u0390\u0001\u0000\u0000\u0000\u00ba\u0392\u0001\u0000\u0000\u0000\u00bc"+
+    "\u0396\u0001\u0000\u0000\u0000\u00be\u039a\u0001\u0000\u0000\u0000\u00c0"+
+    "\u039e\u0001\u0000\u0000\u0000\u00c2\u03a3\u0001\u0000\u0000\u0000\u00c4"+
+    "\u03a7\u0001\u0000\u0000\u0000\u00c6\u03ab\u0001\u0000\u0000\u0000\u00c8"+
+    "\u03af\u0001\u0000\u0000\u0000\u00ca\u03b3\u0001\u0000\u0000\u0000\u00cc"+
+    "\u03b7\u0001\u0000\u0000\u0000\u00ce\u03c0\u0001\u0000\u0000\u0000\u00d0"+
+    "\u03c4\u0001\u0000\u0000\u0000\u00d2\u03c8\u0001\u0000\u0000\u0000\u00d4"+
+    "\u03cc\u0001\u0000\u0000\u0000\u00d6\u03d0\u0001\u0000\u0000\u0000\u00d8"+
+    "\u03d5\u0001\u0000\u0000\u0000\u00da\u03d9\u0001\u0000\u0000\u0000\u00dc"+
+    "\u03e1\u0001\u0000\u0000\u0000\u00de\u03f6\u0001\u0000\u0000\u0000\u00e0"+
+    "\u03fa\u0001\u0000\u0000\u0000\u00e2\u03fe\u0001\u0000\u0000\u0000\u00e4"+
+    "\u0402\u0001\u0000\u0000\u0000\u00e6\u0406\u0001\u0000\u0000\u0000\u00e8"+
+    "\u040a\u0001\u0000\u0000\u0000\u00ea\u040f\u0001\u0000\u0000\u0000\u00ec"+
+    "\u0413\u0001\u0000\u0000\u0000\u00ee\u0417\u0001\u0000\u0000\u0000\u00f0"+
+    "\u041b\u0001\u0000\u0000\u0000\u00f2\u041e\u0001\u0000\u0000\u0000\u00f4"+
+    "\u0422\u0001\u0000\u0000\u0000\u00f6\u0426\u0001\u0000\u0000\u0000\u00f8"+
+    "\u042a\u0001\u0000\u0000\u0000\u00fa\u042e\u0001\u0000\u0000\u0000\u00fc"+
+    "\u0433\u0001\u0000\u0000\u0000\u00fe\u0438\u0001\u0000\u0000\u0000\u0100"+
+    "\u043d\u0001\u0000\u0000\u0000\u0102\u0444\u0001\u0000\u0000\u0000\u0104"+
+    "\u044d\u0001\u0000\u0000\u0000\u0106\u0454\u0001\u0000\u0000\u0000\u0108"+
+    "\u0458\u0001\u0000\u0000\u0000\u010a\u045c\u0001\u0000\u0000\u0000\u010c"+
+    "\u0460\u0001\u0000\u0000\u0000\u010e\u0464\u0001\u0000\u0000\u0000\u0110"+
+    "\u0468\u0001\u0000\u0000\u0000\u0112\u046e\u0001\u0000\u0000\u0000\u0114"+
+    "\u0472\u0001\u0000\u0000\u0000\u0116\u0476\u0001\u0000\u0000\u0000\u0118"+
+    "\u047a\u0001\u0000\u0000\u0000\u011a\u047e\u0001\u0000\u0000\u0000\u011c"+
+    "\u0482\u0001\u0000\u0000\u0000\u011e\u0486\u0001\u0000\u0000\u0000\u0120"+
+    "\u048a\u0001\u0000\u0000\u0000\u0122\u048e\u0001\u0000\u0000\u0000\u0124"+
+    "\u0492\u0001\u0000\u0000\u0000\u0126\u0497\u0001\u0000\u0000\u0000\u0128"+
+    "\u049b\u0001\u0000\u0000\u0000\u012a\u049f\u0001\u0000\u0000\u0000\u012c"+
+    "\u04a4\u0001\u0000\u0000\u0000\u012e\u04a8\u0001\u0000\u0000\u0000\u0130"+
+    "\u04ac\u0001\u0000\u0000\u0000\u0132\u04b0\u0001\u0000\u0000\u0000\u0134"+
+    "\u04b4\u0001\u0000\u0000\u0000\u0136\u04ba\u0001\u0000\u0000\u0000\u0138"+
+    "\u04be\u0001\u0000\u0000\u0000\u013a\u04c2\u0001\u0000\u0000\u0000\u013c"+
+    "\u04c6\u0001\u0000\u0000\u0000\u013e\u04ca\u0001\u0000\u0000\u0000\u0140"+
+    "\u04ce\u0001\u0000\u0000\u0000\u0142\u04d2\u0001\u0000\u0000\u0000\u0144"+
+    "\u04d7\u0001\u0000\u0000\u0000\u0146\u04db\u0001\u0000\u0000\u0000\u0148"+
+    "\u04df\u0001\u0000\u0000\u0000\u014a\u04e3\u0001\u0000\u0000\u0000\u014c"+
+    "\u04e7\u0001\u0000\u0000\u0000\u014e\u04eb\u0001\u0000\u0000\u0000\u0150"+
+    "\u04ef\u0001\u0000\u0000\u0000\u0152\u04f4\u0001\u0000\u0000\u0000\u0154"+
+    "\u04f9\u0001\u0000\u0000\u0000\u0156\u04fd\u0001\u0000\u0000\u0000\u0158"+
+    "\u0501\u0001\u0000\u0000\u0000\u015a\u0505\u0001\u0000\u0000\u0000\u015c"+
+    "\u050a\u0001\u0000\u0000\u0000\u015e\u0514\u0001\u0000\u0000\u0000\u0160"+
+    "\u0518\u0001\u0000\u0000\u0000\u0162\u051c\u0001\u0000\u0000\u0000\u0164"+
+    "\u0520\u0001\u0000\u0000\u0000\u0166\u0525\u0001\u0000\u0000\u0000\u0168"+
+    "\u052c\u0001\u0000\u0000\u0000\u016a\u0530\u0001\u0000\u0000\u0000\u016c"+
+    "\u0534\u0001\u0000\u0000\u0000\u016e\u0538\u0001\u0000\u0000\u0000\u0170"+
+    "\u053c\u0001\u0000\u0000\u0000\u0172\u0541\u0001\u0000\u0000\u0000\u0174"+
+    "\u0547\u0001\u0000\u0000\u0000\u0176\u054b\u0001\u0000\u0000\u0000\u0178"+
+    "\u054f\u0001\u0000\u0000\u0000\u017a\u0553\u0001\u0000\u0000\u0000\u017c"+
+    "\u0559\u0001\u0000\u0000\u0000\u017e\u055d\u0001\u0000\u0000\u0000\u0180"+
+    "\u0561\u0001\u0000\u0000\u0000\u0182\u0565\u0001\u0000\u0000\u0000\u0184"+
+    "\u056b\u0001\u0000\u0000\u0000\u0186\u0571\u0001\u0000\u0000\u0000\u0188"+
+    "\u0577\u0001\u0000\u0000\u0000\u018a\u018b\u0005d\u0000\u0000\u018b\u018c"+
+    "\u0005i\u0000\u0000\u018c\u018d\u0005s\u0000\u0000\u018d\u018e\u0005s"+
+    "\u0000\u0000\u018e\u018f\u0005e\u0000\u0000\u018f\u0190\u0005c\u0000\u0000"+
+    "\u0190\u0191\u0005t\u0000\u0000\u0191\u0192\u0001\u0000\u0000\u0000\u0192"+
+    "\u0193\u0006\u0000\u0000\u0000\u0193\u0011\u0001\u0000\u0000\u0000\u0194"+
+    "\u0195\u0005d\u0000\u0000\u0195\u0196\u0005r\u0000\u0000\u0196\u0197\u0005"+
+    "o\u0000\u0000\u0197\u0198\u0005p\u0000\u0000\u0198\u0199\u0001\u0000\u0000"+
+    "\u0000\u0199\u019a\u0006\u0001\u0001\u0000\u019a\u0013\u0001\u0000\u0000"+
+    "\u0000\u019b\u019c\u0005e\u0000\u0000\u019c\u019d\u0005n\u0000\u0000\u019d"+
+    "\u019e\u0005r\u0000\u0000\u019e\u019f\u0005i\u0000\u0000\u019f\u01a0\u0005"+
+    "c\u0000\u0000\u01a0\u01a1\u0005h\u0000\u0000\u01a1\u01a2\u0001\u0000\u0000"+
+    "\u0000\u01a2\u01a3\u0006\u0002\u0002\u0000\u01a3\u0015\u0001\u0000\u0000"+
+    "\u0000\u01a4\u01a5\u0005e\u0000\u0000\u01a5\u01a6\u0005v\u0000\u0000\u01a6"+
+    "\u01a7\u0005a\u0000\u0000\u01a7\u01a8\u0005l\u0000\u0000\u01a8\u01a9\u0001"+
+    "\u0000\u0000\u0000\u01a9\u01aa\u0006\u0003\u0000\u0000\u01aa\u0017\u0001"+
+    "\u0000\u0000\u0000\u01ab\u01ac\u0005e\u0000\u0000\u01ac\u01ad\u0005x\u0000"+
+    "\u0000\u01ad\u01ae\u0005p\u0000\u0000\u01ae\u01af\u0005l\u0000\u0000\u01af"+
+    "\u01b0\u0005a\u0000\u0000\u01b0\u01b1\u0005i\u0000\u0000\u01b1\u01b2\u0005"+
+    "n\u0000\u0000\u01b2\u01b3\u0001\u0000\u0000\u0000\u01b3\u01b4\u0006\u0004"+
+    "\u0003\u0000\u01b4\u0019\u0001\u0000\u0000\u0000\u01b5\u01b6\u0005f\u0000"+
+    "\u0000\u01b6\u01b7\u0005r\u0000\u0000\u01b7\u01b8\u0005o\u0000\u0000\u01b8"+
+    "\u01b9\u0005m\u0000\u0000\u01b9\u01ba\u0001\u0000\u0000\u0000\u01ba\u01bb"+
+    "\u0006\u0005\u0004\u0000\u01bb\u001b\u0001\u0000\u0000\u0000\u01bc\u01bd"+
+    "\u0005g\u0000\u0000\u01bd\u01be\u0005r\u0000\u0000\u01be\u01bf\u0005o"+
+    "\u0000\u0000\u01bf\u01c0\u0005k\u0000\u0000\u01c0\u01c1\u0001\u0000\u0000"+
+    "\u0000\u01c1\u01c2\u0006\u0006\u0000\u0000\u01c2\u001d\u0001\u0000\u0000"+
+    "\u0000\u01c3\u01c4\u0005i\u0000\u0000\u01c4\u01c5\u0005n\u0000\u0000\u01c5"+
+    "\u01c6\u0005l\u0000\u0000\u01c6\u01c7\u0005i\u0000\u0000\u01c7\u01c8\u0005"+
+    "n\u0000\u0000\u01c8\u01c9\u0005e\u0000\u0000\u01c9\u01ca\u0005s\u0000"+
+    "\u0000\u01ca\u01cb\u0005t\u0000\u0000\u01cb\u01cc\u0005a\u0000\u0000\u01cc"+
+    "\u01cd\u0005t\u0000\u0000\u01cd\u01ce\u0005s\u0000\u0000\u01ce\u01cf\u0001"+
+    "\u0000\u0000\u0000\u01cf\u01d0\u0006\u0007\u0000\u0000\u01d0\u001f\u0001"+
+    "\u0000\u0000\u0000\u01d1\u01d2\u0005k\u0000\u0000\u01d2\u01d3\u0005e\u0000"+
+    "\u0000\u01d3\u01d4\u0005e\u0000\u0000\u01d4\u01d5\u0005p\u0000\u0000\u01d5"+
+    "\u01d6\u0001\u0000\u0000\u0000\u01d6\u01d7\u0006\b\u0001\u0000\u01d7!"+
+    "\u0001\u0000\u0000\u0000\u01d8\u01d9\u0005l\u0000\u0000\u01d9\u01da\u0005"+
+    "i\u0000\u0000\u01da\u01db\u0005m\u0000\u0000\u01db\u01dc\u0005i\u0000"+
+    "\u0000\u01dc\u01dd\u0005t\u0000\u0000\u01dd\u01de\u0001\u0000\u0000\u0000"+
+    "\u01de\u01df\u0006\t\u0000\u0000\u01df#\u0001\u0000\u0000\u0000\u01e0"+
+    "\u01e1\u0005l\u0000\u0000\u01e1\u01e2\u0005o\u0000\u0000\u01e2\u01e3\u0005"+
+    "o\u0000\u0000\u01e3\u01e4\u0005k\u0000\u0000\u01e4\u01e5\u0005u\u0000"+
+    "\u0000\u01e5\u01e6\u0005p\u0000\u0000\u01e6\u01e7\u0001\u0000\u0000\u0000"+
+    "\u01e7\u01e8\u0006\n\u0005\u0000\u01e8%\u0001\u0000\u0000\u0000\u01e9"+
+    "\u01ea\u0005m\u0000\u0000\u01ea\u01eb\u0005e\u0000\u0000\u01eb\u01ec\u0005"+
+    "t\u0000\u0000\u01ec\u01ed\u0005a\u0000\u0000\u01ed\u01ee\u0001\u0000\u0000"+
+    "\u0000\u01ee\u01ef\u0006\u000b\u0006\u0000\u01ef\'\u0001\u0000\u0000\u0000"+
+    "\u01f0\u01f1\u0005m\u0000\u0000\u01f1\u01f2\u0005e\u0000\u0000\u01f2\u01f3"+
+    "\u0005t\u0000\u0000\u01f3\u01f4\u0005r\u0000\u0000\u01f4\u01f5\u0005i"+
+    "\u0000\u0000\u01f5\u01f6\u0005c\u0000\u0000\u01f6\u01f7\u0005s\u0000\u0000"+
+    "\u01f7\u01f8\u0001\u0000\u0000\u0000\u01f8\u01f9\u0006\f\u0007\u0000\u01f9"+
+    ")\u0001\u0000\u0000\u0000\u01fa\u01fb\u0005m\u0000\u0000\u01fb\u01fc\u0005"+
+    "v\u0000\u0000\u01fc\u01fd\u0005_\u0000\u0000\u01fd\u01fe\u0005e\u0000"+
+    "\u0000\u01fe\u01ff\u0005x\u0000\u0000\u01ff\u0200\u0005p\u0000\u0000\u0200"+
+    "\u0201\u0005a\u0000\u0000\u0201\u0202\u0005n\u0000\u0000\u0202\u0203\u0005"+
+    "d\u0000\u0000\u0203\u0204\u0001\u0000\u0000\u0000\u0204\u0205\u0006\r"+
+    "\b\u0000\u0205+\u0001\u0000\u0000\u0000\u0206\u0207\u0005r\u0000\u0000"+
+    "\u0207\u0208\u0005e\u0000\u0000\u0208\u0209\u0005n\u0000\u0000\u0209\u020a"+
+    "\u0005a\u0000\u0000\u020a\u020b\u0005m\u0000\u0000\u020b\u020c\u0005e"+
+    "\u0000\u0000\u020c\u020d\u0001\u0000\u0000\u0000\u020d\u020e\u0006\u000e"+
+    "\t\u0000\u020e-\u0001\u0000\u0000\u0000\u020f\u0210\u0005r\u0000\u0000"+
+    "\u0210\u0211\u0005o\u0000\u0000\u0211\u0212\u0005w\u0000\u0000\u0212\u0213"+
+    "\u0001\u0000\u0000\u0000\u0213\u0214\u0006\u000f\u0000\u0000\u0214/\u0001"+
+    "\u0000\u0000\u0000\u0215\u0216\u0005s\u0000\u0000\u0216\u0217\u0005h\u0000"+
+    "\u0000\u0217\u0218\u0005o\u0000\u0000\u0218\u0219\u0005w\u0000\u0000\u0219"+
+    "\u021a\u0001\u0000\u0000\u0000\u021a\u021b\u0006\u0010\n\u0000\u021b1"+
+    "\u0001\u0000\u0000\u0000\u021c\u021d\u0005s\u0000\u0000\u021d\u021e\u0005"+
+    "o\u0000\u0000\u021e\u021f\u0005r\u0000\u0000\u021f\u0220\u0005t\u0000"+
+    "\u0000\u0220\u0221\u0001\u0000\u0000\u0000\u0221\u0222\u0006\u0011\u0000"+
+    "\u0000\u02223\u0001\u0000\u0000\u0000\u0223\u0224\u0005s\u0000\u0000\u0224"+
+    "\u0225\u0005t\u0000\u0000\u0225\u0226\u0005a\u0000\u0000\u0226\u0227\u0005"+
+    "t\u0000\u0000\u0227\u0228\u0005s\u0000\u0000\u0228\u0229\u0001\u0000\u0000"+
+    "\u0000\u0229\u022a\u0006\u0012\u0000\u0000\u022a5\u0001\u0000\u0000\u0000"+
+    "\u022b\u022c\u0005w\u0000\u0000\u022c\u022d\u0005h\u0000\u0000\u022d\u022e"+
+    "\u0005e\u0000\u0000\u022e\u022f\u0005r\u0000\u0000\u022f\u0230\u0005e"+
+    "\u0000\u0000\u0230\u0231\u0001\u0000\u0000\u0000\u0231\u0232\u0006\u0013"+
+    "\u0000\u0000\u02327\u0001\u0000\u0000\u0000\u0233\u0235\b\u0000\u0000"+
+    "\u0000\u0234\u0233\u0001\u0000\u0000\u0000\u0235\u0236\u0001\u0000\u0000"+
+    "\u0000\u0236\u0234\u0001\u0000\u0000\u0000\u0236\u0237\u0001\u0000\u0000"+
+    "\u0000\u0237\u0238\u0001\u0000\u0000\u0000\u0238\u0239\u0006\u0014\u0000"+
+    "\u0000\u02399\u0001\u0000\u0000\u0000\u023a\u023b\u0005/\u0000\u0000\u023b"+
+    "\u023c\u0005/\u0000\u0000\u023c\u0240\u0001\u0000\u0000\u0000\u023d\u023f"+
+    "\b\u0001\u0000\u0000\u023e\u023d\u0001\u0000\u0000\u0000\u023f\u0242\u0001"+
+    "\u0000\u0000\u0000\u0240\u023e\u0001\u0000\u0000\u0000\u0240\u0241\u0001"+
+    "\u0000\u0000\u0000\u0241\u0244\u0001\u0000\u0000\u0000\u0242\u0240\u0001"+
+    "\u0000\u0000\u0000\u0243\u0245\u0005\r\u0000\u0000\u0244\u0243\u0001\u0000"+
+    "\u0000\u0000\u0244\u0245\u0001\u0000\u0000\u0000\u0245\u0247\u0001\u0000"+
+    "\u0000\u0000\u0246\u0248\u0005\n\u0000\u0000\u0247\u0246\u0001\u0000\u0000"+
+    "\u0000\u0247\u0248\u0001\u0000\u0000\u0000\u0248\u0249\u0001\u0000\u0000"+
+    "\u0000\u0249\u024a\u0006\u0015\u000b\u0000\u024a;\u0001\u0000\u0000\u0000"+
+    "\u024b\u024c\u0005/\u0000\u0000\u024c\u024d\u0005*\u0000\u0000\u024d\u0252"+
+    "\u0001\u0000\u0000\u0000\u024e\u0251\u0003<\u0016\u0000\u024f\u0251\t"+
+    "\u0000\u0000\u0000\u0250\u024e\u0001\u0000\u0000\u0000\u0250\u024f\u0001"+
+    "\u0000\u0000\u0000\u0251\u0254\u0001\u0000\u0000\u0000\u0252\u0253\u0001"+
+    "\u0000\u0000\u0000\u0252\u0250\u0001\u0000\u0000\u0000\u0253\u0255\u0001"+
+    "\u0000\u0000\u0000\u0254\u0252\u0001\u0000\u0000\u0000\u0255\u0256\u0005"+
+    "*\u0000\u0000\u0256\u0257\u0005/\u0000\u0000\u0257\u0258\u0001\u0000\u0000"+
+    "\u0000\u0258\u0259\u0006\u0016\u000b\u0000\u0259=\u0001\u0000\u0000\u0000"+
+    "\u025a\u025c\u0007\u0002\u0000\u0000\u025b\u025a\u0001\u0000\u0000\u0000"+
+    "\u025c\u025d\u0001\u0000\u0000\u0000\u025d\u025b\u0001\u0000\u0000\u0000"+
+    "\u025d\u025e\u0001\u0000\u0000\u0000\u025e\u025f\u0001\u0000\u0000\u0000"+
+    "\u025f\u0260\u0006\u0017\u000b\u0000\u0260?\u0001\u0000\u0000\u0000\u0261"+
+    "\u0265\b\u0003\u0000\u0000\u0262\u0263\u0005/\u0000\u0000\u0263\u0265"+
+    "\b\u0004\u0000\u0000\u0264\u0261\u0001\u0000\u0000\u0000\u0264\u0262\u0001"+
+    "\u0000\u0000\u0000\u0265A\u0001\u0000\u0000\u0000\u0266\u0268\u0003@\u0018"+
+    "\u0000\u0267\u0266\u0001\u0000\u0000\u0000\u0268\u0269\u0001\u0000\u0000"+
+    "\u0000\u0269\u0267\u0001\u0000\u0000\u0000\u0269\u026a\u0001\u0000\u0000"+
+    "\u0000\u026aC\u0001\u0000\u0000\u0000\u026b\u026c\u0003\u00b0P\u0000\u026c"+
+    "\u026d\u0001\u0000\u0000\u0000\u026d\u026e\u0006\u001a\f\u0000\u026e\u026f"+
+    "\u0006\u001a\r\u0000\u026fE\u0001\u0000\u0000\u0000\u0270\u0271\u0003"+
+    "N\u001f\u0000\u0271\u0272\u0001\u0000\u0000\u0000\u0272\u0273\u0006\u001b"+
+    "\u000e\u0000\u0273\u0274\u0006\u001b\u000f\u0000\u0274G\u0001\u0000\u0000"+
+    "\u0000\u0275\u0276\u0003>\u0017\u0000\u0276\u0277\u0001\u0000\u0000\u0000"+
+    "\u0277\u0278\u0006\u001c\u000b\u0000\u0278I\u0001\u0000\u0000\u0000\u0279"+
+    "\u027a\u0003:\u0015\u0000\u027a\u027b\u0001\u0000\u0000\u0000\u027b\u027c"+
+    "\u0006\u001d\u000b\u0000\u027cK\u0001\u0000\u0000\u0000\u027d\u027e\u0003"+
+    "<\u0016\u0000\u027e\u027f\u0001\u0000\u0000\u0000\u027f\u0280\u0006\u001e"+
+    "\u000b\u0000\u0280M\u0001\u0000\u0000\u0000\u0281\u0282\u0005|\u0000\u0000"+
+    "\u0282\u0283\u0001\u0000\u0000\u0000\u0283\u0284\u0006\u001f\u000f\u0000"+
+    "\u0284O\u0001\u0000\u0000\u0000\u0285\u0286\u0007\u0005\u0000\u0000\u0286"+
+    "Q\u0001\u0000\u0000\u0000\u0287\u0288\u0007\u0006\u0000\u0000\u0288S\u0001"+
+    "\u0000\u0000\u0000\u0289\u028a\u0005\\\u0000\u0000\u028a\u028b\u0007\u0007"+
+    "\u0000\u0000\u028bU\u0001\u0000\u0000\u0000\u028c\u028d\b\b\u0000\u0000"+
+    "\u028dW\u0001\u0000\u0000\u0000\u028e\u0290\u0007\t\u0000\u0000\u028f"+
+    "\u0291\u0007\n\u0000\u0000\u0290\u028f\u0001\u0000\u0000\u0000\u0290\u0291"+
+    "\u0001\u0000\u0000\u0000\u0291\u0293\u0001\u0000\u0000\u0000\u0292\u0294"+
+    "\u0003P \u0000\u0293\u0292\u0001\u0000\u0000\u0000\u0294\u0295\u0001\u0000"+
+    "\u0000\u0000\u0295\u0293\u0001\u0000\u0000\u0000\u0295\u0296\u0001\u0000"+
+    "\u0000\u0000\u0296Y\u0001\u0000\u0000\u0000\u0297\u0298\u0005@\u0000\u0000"+
+    "\u0298[\u0001\u0000\u0000\u0000\u0299\u029a\u0005`\u0000\u0000\u029a]"+
+    "\u0001\u0000\u0000\u0000\u029b\u029f\b\u000b\u0000\u0000\u029c\u029d\u0005"+
+    "`\u0000\u0000\u029d\u029f\u0005`\u0000\u0000\u029e\u029b\u0001\u0000\u0000"+
+    "\u0000\u029e\u029c\u0001\u0000\u0000\u0000\u029f_\u0001\u0000\u0000\u0000"+
+    "\u02a0\u02a1\u0005_\u0000\u0000\u02a1a\u0001\u0000\u0000\u0000\u02a2\u02a6"+
+    "\u0003R!\u0000\u02a3\u02a6\u0003P \u0000\u02a4\u02a6\u0003`(\u0000\u02a5"+
+    "\u02a2\u0001\u0000\u0000\u0000\u02a5\u02a3\u0001\u0000\u0000\u0000\u02a5"+
+    "\u02a4\u0001\u0000\u0000\u0000\u02a6c\u0001\u0000\u0000\u0000\u02a7\u02ac"+
+    "\u0005\"\u0000\u0000\u02a8\u02ab\u0003T\"\u0000\u02a9\u02ab\u0003V#\u0000"+
+    "\u02aa\u02a8\u0001\u0000\u0000\u0000\u02aa\u02a9\u0001\u0000\u0000\u0000"+
+    "\u02ab\u02ae\u0001\u0000\u0000\u0000\u02ac\u02aa\u0001\u0000\u0000\u0000"+
+    "\u02ac\u02ad\u0001\u0000\u0000\u0000\u02ad\u02af\u0001\u0000\u0000\u0000"+
+    "\u02ae\u02ac\u0001\u0000\u0000\u0000\u02af\u02c5\u0005\"\u0000\u0000\u02b0"+
+    "\u02b1\u0005\"\u0000\u0000\u02b1\u02b2\u0005\"\u0000\u0000\u02b2\u02b3"+
+    "\u0005\"\u0000\u0000\u02b3\u02b7\u0001\u0000\u0000\u0000\u02b4\u02b6\b"+
+    "\u0001\u0000\u0000\u02b5\u02b4\u0001\u0000\u0000\u0000\u02b6\u02b9\u0001"+
+    "\u0000\u0000\u0000\u02b7\u02b8\u0001\u0000\u0000\u0000\u02b7\u02b5\u0001"+
+    "\u0000\u0000\u0000\u02b8\u02ba\u0001\u0000\u0000\u0000\u02b9\u02b7\u0001"+
+    "\u0000\u0000\u0000\u02ba\u02bb\u0005\"\u0000\u0000\u02bb\u02bc\u0005\""+
+    "\u0000\u0000\u02bc\u02bd\u0005\"\u0000\u0000\u02bd\u02bf\u0001\u0000\u0000"+
+    "\u0000\u02be\u02c0\u0005\"\u0000\u0000\u02bf\u02be\u0001\u0000\u0000\u0000"+
+    "\u02bf\u02c0\u0001\u0000\u0000\u0000\u02c0\u02c2\u0001\u0000\u0000\u0000"+
+    "\u02c1\u02c3\u0005\"\u0000\u0000\u02c2\u02c1\u0001\u0000\u0000\u0000\u02c2"+
+    "\u02c3\u0001\u0000\u0000\u0000\u02c3\u02c5\u0001\u0000\u0000\u0000\u02c4"+
+    "\u02a7\u0001\u0000\u0000\u0000\u02c4\u02b0\u0001\u0000\u0000\u0000\u02c5"+
+    "e\u0001\u0000\u0000\u0000\u02c6\u02c8\u0003P \u0000\u02c7\u02c6\u0001"+
+    "\u0000\u0000\u0000\u02c8\u02c9\u0001\u0000\u0000\u0000\u02c9\u02c7\u0001"+
+    "\u0000\u0000\u0000\u02c9\u02ca\u0001\u0000\u0000\u0000\u02cag\u0001\u0000"+
+    "\u0000\u0000\u02cb\u02cd\u0003P \u0000\u02cc\u02cb\u0001\u0000\u0000\u0000"+
+    "\u02cd\u02ce\u0001\u0000\u0000\u0000\u02ce\u02cc\u0001\u0000\u0000\u0000"+
+    "\u02ce\u02cf\u0001\u0000\u0000\u0000\u02cf\u02d0\u0001\u0000\u0000\u0000"+
+    "\u02d0\u02d4\u0003x4\u0000\u02d1\u02d3\u0003P \u0000\u02d2\u02d1\u0001"+
+    "\u0000\u0000\u0000\u02d3\u02d6\u0001\u0000\u0000\u0000\u02d4\u02d2\u0001"+
+    "\u0000\u0000\u0000\u02d4\u02d5\u0001\u0000\u0000\u0000\u02d5\u02f6\u0001"+
+    "\u0000\u0000\u0000\u02d6\u02d4\u0001\u0000\u0000\u0000\u02d7\u02d9\u0003"+
+    "x4\u0000\u02d8\u02da\u0003P \u0000\u02d9\u02d8\u0001\u0000\u0000\u0000"+
+    "\u02da\u02db\u0001\u0000\u0000\u0000\u02db\u02d9\u0001\u0000\u0000\u0000"+
+    "\u02db\u02dc\u0001\u0000\u0000\u0000\u02dc\u02f6\u0001\u0000\u0000\u0000"+
+    "\u02dd\u02df\u0003P \u0000\u02de\u02dd\u0001\u0000\u0000\u0000\u02df\u02e0"+
+    "\u0001\u0000\u0000\u0000\u02e0\u02de\u0001\u0000\u0000\u0000\u02e0\u02e1"+
+    "\u0001\u0000\u0000\u0000\u02e1\u02e9\u0001\u0000\u0000\u0000\u02e2\u02e6"+
+    "\u0003x4\u0000\u02e3\u02e5\u0003P \u0000\u02e4\u02e3\u0001\u0000\u0000"+
+    "\u0000\u02e5\u02e8\u0001\u0000\u0000\u0000\u02e6\u02e4\u0001\u0000\u0000"+
+    "\u0000\u02e6\u02e7\u0001\u0000\u0000\u0000\u02e7\u02ea\u0001\u0000\u0000"+
+    "\u0000\u02e8\u02e6\u0001\u0000\u0000\u0000\u02e9\u02e2\u0001\u0000\u0000"+
+    "\u0000\u02e9\u02ea\u0001\u0000\u0000\u0000\u02ea\u02eb\u0001\u0000\u0000"+
+    "\u0000\u02eb\u02ec\u0003X$\u0000\u02ec\u02f6\u0001\u0000\u0000\u0000\u02ed"+
+    "\u02ef\u0003x4\u0000\u02ee\u02f0\u0003P \u0000\u02ef\u02ee\u0001\u0000"+
+    "\u0000\u0000\u02f0\u02f1\u0001\u0000\u0000\u0000\u02f1\u02ef\u0001\u0000"+
+    "\u0000\u0000\u02f1\u02f2\u0001\u0000\u0000\u0000\u02f2\u02f3\u0001\u0000"+
+    "\u0000\u0000\u02f3\u02f4\u0003X$\u0000\u02f4\u02f6\u0001\u0000\u0000\u0000"+
+    "\u02f5\u02cc\u0001\u0000\u0000\u0000\u02f5\u02d7\u0001\u0000\u0000\u0000"+
+    "\u02f5\u02de\u0001\u0000\u0000\u0000\u02f5\u02ed\u0001\u0000\u0000\u0000"+
+    "\u02f6i\u0001\u0000\u0000\u0000\u02f7\u02f8\u0005b\u0000\u0000\u02f8\u02f9"+
+    "\u0005y\u0000\u0000\u02f9k\u0001\u0000\u0000\u0000\u02fa\u02fb\u0005a"+
+    "\u0000\u0000\u02fb\u02fc\u0005n\u0000\u0000\u02fc\u02fd\u0005d\u0000\u0000"+
+    "\u02fdm\u0001\u0000\u0000\u0000\u02fe\u02ff\u0005a\u0000\u0000\u02ff\u0300"+
+    "\u0005s\u0000\u0000\u0300\u0301\u0005c\u0000\u0000\u0301o\u0001\u0000"+
+    "\u0000\u0000\u0302\u0303\u0005=\u0000\u0000\u0303q\u0001\u0000\u0000\u0000"+
+    "\u0304\u0305\u0005:\u0000\u0000\u0305\u0306\u0005:\u0000\u0000\u0306s"+
+    "\u0001\u0000\u0000\u0000\u0307\u0308\u0005,\u0000\u0000\u0308u\u0001\u0000"+
+    "\u0000\u0000\u0309\u030a\u0005d\u0000\u0000\u030a\u030b\u0005e\u0000\u0000"+
+    "\u030b\u030c\u0005s\u0000\u0000\u030c\u030d\u0005c\u0000\u0000\u030dw"+
+    "\u0001\u0000\u0000\u0000\u030e\u030f\u0005.\u0000\u0000\u030fy\u0001\u0000"+
+    "\u0000\u0000\u0310\u0311\u0005f\u0000\u0000\u0311\u0312\u0005a\u0000\u0000"+
+    "\u0312\u0313\u0005l\u0000\u0000\u0313\u0314\u0005s\u0000\u0000\u0314\u0315"+
+    "\u0005e\u0000\u0000\u0315{\u0001\u0000\u0000\u0000\u0316\u0317\u0005f"+
+    "\u0000\u0000\u0317\u0318\u0005i\u0000\u0000\u0318\u0319\u0005r\u0000\u0000"+
+    "\u0319\u031a\u0005s\u0000\u0000\u031a\u031b\u0005t\u0000\u0000\u031b}"+
+    "\u0001\u0000\u0000\u0000\u031c\u031d\u0005l\u0000\u0000\u031d\u031e\u0005"+
+    "a\u0000\u0000\u031e\u031f\u0005s\u0000\u0000\u031f\u0320\u0005t\u0000"+
+    "\u0000\u0320\u007f\u0001\u0000\u0000\u0000\u0321\u0322\u0005(\u0000\u0000"+
+    "\u0322\u0081\u0001\u0000\u0000\u0000\u0323\u0324\u0005i\u0000\u0000\u0324"+
+    "\u0325\u0005n\u0000\u0000\u0325\u0083\u0001\u0000\u0000\u0000\u0326\u0327"+
+    "\u0005i\u0000\u0000\u0327\u0328\u0005s\u0000\u0000\u0328\u0085\u0001\u0000"+
+    "\u0000\u0000\u0329\u032a\u0005l\u0000\u0000\u032a\u032b\u0005i\u0000\u0000"+
+    "\u032b\u032c\u0005k\u0000\u0000\u032c\u032d\u0005e\u0000\u0000\u032d\u0087"+
+    "\u0001\u0000\u0000\u0000\u032e\u032f\u0005n\u0000\u0000\u032f\u0330\u0005"+
+    "o\u0000\u0000\u0330\u0331\u0005t\u0000\u0000\u0331\u0089\u0001\u0000\u0000"+
+    "\u0000\u0332\u0333\u0005n\u0000\u0000\u0333\u0334\u0005u\u0000\u0000\u0334"+
+    "\u0335\u0005l\u0000\u0000\u0335\u0336\u0005l\u0000\u0000\u0336\u008b\u0001"+
+    "\u0000\u0000\u0000\u0337\u0338\u0005n\u0000\u0000\u0338\u0339\u0005u\u0000"+
+    "\u0000\u0339\u033a\u0005l\u0000\u0000\u033a\u033b\u0005l\u0000\u0000\u033b"+
+    "\u033c\u0005s\u0000\u0000\u033c\u008d\u0001\u0000\u0000\u0000\u033d\u033e"+
+    "\u0005o\u0000\u0000\u033e\u033f\u0005r\u0000\u0000\u033f\u008f\u0001\u0000"+
+    "\u0000\u0000\u0340\u0341\u0005?\u0000\u0000\u0341\u0091\u0001\u0000\u0000"+
+    "\u0000\u0342\u0343\u0005r\u0000\u0000\u0343\u0344\u0005l\u0000\u0000\u0344"+
+    "\u0345\u0005i\u0000\u0000\u0345\u0346\u0005k\u0000\u0000\u0346\u0347\u0005"+
+    "e\u0000\u0000\u0347\u0093\u0001\u0000\u0000\u0000\u0348\u0349\u0005)\u0000"+
+    "\u0000\u0349\u0095\u0001\u0000\u0000\u0000\u034a\u034b\u0005t\u0000\u0000"+
+    "\u034b\u034c\u0005r\u0000\u0000\u034c\u034d\u0005u\u0000\u0000\u034d\u034e"+
+    "\u0005e\u0000\u0000\u034e\u0097\u0001\u0000\u0000\u0000\u034f\u0350\u0005"+
+    "=\u0000\u0000\u0350\u0351\u0005=\u0000\u0000\u0351\u0099\u0001\u0000\u0000"+
+    "\u0000\u0352\u0353\u0005=\u0000\u0000\u0353\u0354\u0005~\u0000\u0000\u0354"+
+    "\u009b\u0001\u0000\u0000\u0000\u0355\u0356\u0005!\u0000\u0000\u0356\u0357"+
+    "\u0005=\u0000\u0000\u0357\u009d\u0001\u0000\u0000\u0000\u0358\u0359\u0005"+
+    "<\u0000\u0000\u0359\u009f\u0001\u0000\u0000\u0000\u035a\u035b\u0005<\u0000"+
+    "\u0000\u035b\u035c\u0005=\u0000\u0000\u035c\u00a1\u0001\u0000\u0000\u0000"+
+    "\u035d\u035e\u0005>\u0000\u0000\u035e\u00a3\u0001\u0000\u0000\u0000\u035f"+
+    "\u0360\u0005>\u0000\u0000\u0360\u0361\u0005=\u0000\u0000\u0361\u00a5\u0001"+
+    "\u0000\u0000\u0000\u0362\u0363\u0005+\u0000\u0000\u0363\u00a7\u0001\u0000"+
+    "\u0000\u0000\u0364\u0365\u0005-\u0000\u0000\u0365\u00a9\u0001\u0000\u0000"+
+    "\u0000\u0366\u0367\u0005*\u0000\u0000\u0367\u00ab\u0001\u0000\u0000\u0000"+
+    "\u0368\u0369\u0005/\u0000\u0000\u0369\u00ad\u0001\u0000\u0000\u0000\u036a"+
+    "\u036b\u0005%\u0000\u0000\u036b\u00af\u0001\u0000\u0000\u0000\u036c\u036d"+
+    "\u0005[\u0000\u0000\u036d\u036e\u0001\u0000\u0000\u0000\u036e\u036f\u0006"+
+    "P\u0000\u0000\u036f\u0370\u0006P\u0000\u0000\u0370\u00b1\u0001\u0000\u0000"+
+    "\u0000\u0371\u0372\u0005]\u0000\u0000\u0372\u0373\u0001\u0000\u0000\u0000"+
+    "\u0373\u0374\u0006Q\u000f\u0000\u0374\u0375\u0006Q\u000f\u0000\u0375\u00b3"+
+    "\u0001\u0000\u0000\u0000\u0376\u037a\u0003R!\u0000\u0377\u0379\u0003b"+
+    ")\u0000\u0378\u0377\u0001\u0000\u0000\u0000\u0379\u037c\u0001\u0000\u0000"+
+    "\u0000\u037a\u0378\u0001\u0000\u0000\u0000\u037a\u037b\u0001\u0000\u0000"+
+    "\u0000\u037b\u0387\u0001\u0000\u0000\u0000\u037c\u037a\u0001\u0000\u0000"+
+    "\u0000\u037d\u0380\u0003`(\u0000\u037e\u0380\u0003Z%\u0000\u037f\u037d"+
+    "\u0001\u0000\u0000\u0000\u037f\u037e\u0001\u0000\u0000\u0000\u0380\u0382"+
+    "\u0001\u0000\u0000\u0000\u0381\u0383\u0003b)\u0000\u0382\u0381\u0001\u0000"+
+    "\u0000\u0000\u0383\u0384\u0001\u0000\u0000\u0000\u0384\u0382\u0001\u0000"+
+    "\u0000\u0000\u0384\u0385\u0001\u0000\u0000\u0000\u0385\u0387\u0001\u0000"+
+    "\u0000\u0000\u0386\u0376\u0001\u0000\u0000\u0000\u0386\u037f\u0001\u0000"+
+    "\u0000\u0000\u0387\u00b5\u0001\u0000\u0000\u0000\u0388\u038a\u0003\\&"+
+    "\u0000\u0389\u038b\u0003^\'\u0000\u038a\u0389\u0001\u0000\u0000\u0000"+
+    "\u038b\u038c\u0001\u0000\u0000\u0000\u038c\u038a\u0001\u0000\u0000\u0000"+
+    "\u038c\u038d\u0001\u0000\u0000\u0000\u038d\u038e\u0001\u0000\u0000\u0000"+
+    "\u038e\u038f\u0003\\&\u0000\u038f\u00b7\u0001\u0000\u0000\u0000\u0390"+
+    "\u0391\u0003\u00b6S\u0000\u0391\u00b9\u0001\u0000\u0000\u0000\u0392\u0393"+
+    "\u0003:\u0015\u0000\u0393\u0394\u0001\u0000\u0000\u0000\u0394\u0395\u0006"+
+    "U\u000b\u0000\u0395\u00bb\u0001\u0000\u0000\u0000\u0396\u0397\u0003<\u0016"+
+    "\u0000\u0397\u0398\u0001\u0000\u0000\u0000\u0398\u0399\u0006V\u000b\u0000"+
+    "\u0399\u00bd\u0001\u0000\u0000\u0000\u039a\u039b\u0003>\u0017\u0000\u039b"+
+    "\u039c\u0001\u0000\u0000\u0000\u039c\u039d\u0006W\u000b\u0000\u039d\u00bf"+
+    "\u0001\u0000\u0000\u0000\u039e\u039f\u0003N\u001f\u0000\u039f\u03a0\u0001"+
+    "\u0000\u0000\u0000\u03a0\u03a1\u0006X\u000e\u0000\u03a1\u03a2\u0006X\u000f"+
+    "\u0000\u03a2\u00c1\u0001\u0000\u0000\u0000\u03a3\u03a4\u0003\u00b0P\u0000"+
+    "\u03a4\u03a5\u0001\u0000\u0000\u0000\u03a5\u03a6\u0006Y\f\u0000\u03a6"+
+    "\u00c3\u0001\u0000\u0000\u0000\u03a7\u03a8\u0003\u00b2Q\u0000\u03a8\u03a9"+
+    "\u0001\u0000\u0000\u0000\u03a9\u03aa\u0006Z\u0010\u0000\u03aa\u00c5\u0001"+
+    "\u0000\u0000\u0000\u03ab\u03ac\u0003t2\u0000\u03ac\u03ad\u0001\u0000\u0000"+
+    "\u0000\u03ad\u03ae\u0006[\u0011\u0000\u03ae\u00c7\u0001\u0000\u0000\u0000"+
+    "\u03af\u03b0\u0003p0\u0000\u03b0\u03b1\u0001\u0000\u0000\u0000\u03b1\u03b2"+
+    "\u0006\\\u0012\u0000\u03b2\u00c9\u0001\u0000\u0000\u0000\u03b3\u03b4\u0003"+
+    "d*\u0000\u03b4\u03b5\u0001\u0000\u0000\u0000\u03b5\u03b6\u0006]\u0013"+
+    "\u0000\u03b6\u00cb\u0001\u0000\u0000\u0000\u03b7\u03b8\u0005m\u0000\u0000"+
+    "\u03b8\u03b9\u0005e\u0000\u0000\u03b9\u03ba\u0005t\u0000\u0000\u03ba\u03bb"+
+    "\u0005a\u0000\u0000\u03bb\u03bc\u0005d\u0000\u0000\u03bc\u03bd\u0005a"+
+    "\u0000\u0000\u03bd\u03be\u0005t\u0000\u0000\u03be\u03bf\u0005a\u0000\u0000"+
+    "\u03bf\u00cd\u0001\u0000\u0000\u0000\u03c0\u03c1\u0003B\u0019\u0000\u03c1"+
+    "\u03c2\u0001\u0000\u0000\u0000\u03c2\u03c3\u0006_\u0014\u0000\u03c3\u00cf"+
+    "\u0001\u0000\u0000\u0000\u03c4\u03c5\u0003:\u0015\u0000\u03c5\u03c6\u0001"+
+    "\u0000\u0000\u0000\u03c6\u03c7\u0006`\u000b\u0000\u03c7\u00d1\u0001\u0000"+
+    "\u0000\u0000\u03c8\u03c9\u0003<\u0016\u0000\u03c9\u03ca\u0001\u0000\u0000"+
+    "\u0000\u03ca\u03cb\u0006a\u000b\u0000\u03cb\u00d3\u0001\u0000\u0000\u0000"+
+    "\u03cc\u03cd\u0003>\u0017\u0000\u03cd\u03ce\u0001\u0000\u0000\u0000\u03ce"+
+    "\u03cf\u0006b\u000b\u0000\u03cf\u00d5\u0001\u0000\u0000\u0000\u03d0\u03d1"+
+    "\u0003N\u001f\u0000\u03d1\u03d2\u0001\u0000\u0000\u0000\u03d2\u03d3\u0006"+
+    "c\u000e\u0000\u03d3\u03d4\u0006c\u000f\u0000\u03d4\u00d7\u0001\u0000\u0000"+
+    "\u0000\u03d5\u03d6\u0003x4\u0000\u03d6\u03d7\u0001\u0000\u0000\u0000\u03d7"+
+    "\u03d8\u0006d\u0015\u0000\u03d8\u00d9\u0001\u0000\u0000\u0000\u03d9\u03da"+
+    "\u0003t2\u0000\u03da\u03db\u0001\u0000\u0000\u0000\u03db\u03dc\u0006e"+
+    "\u0011\u0000\u03dc\u00db\u0001\u0000\u0000\u0000\u03dd\u03e2\u0003R!\u0000"+
+    "\u03de\u03e2\u0003P \u0000\u03df\u03e2\u0003`(\u0000\u03e0\u03e2\u0003"+
+    "\u00aaM\u0000\u03e1\u03dd\u0001\u0000\u0000\u0000\u03e1\u03de\u0001\u0000"+
+    "\u0000\u0000\u03e1\u03df\u0001\u0000\u0000\u0000\u03e1\u03e0\u0001\u0000"+
+    "\u0000\u0000\u03e2\u00dd\u0001\u0000\u0000\u0000\u03e3\u03e6\u0003R!\u0000"+
+    "\u03e4\u03e6\u0003\u00aaM\u0000\u03e5\u03e3\u0001\u0000\u0000\u0000\u03e5"+
+    "\u03e4\u0001\u0000\u0000\u0000\u03e6\u03ea\u0001\u0000\u0000\u0000\u03e7"+
+    "\u03e9\u0003\u00dcf\u0000\u03e8\u03e7\u0001\u0000\u0000\u0000\u03e9\u03ec"+
+    "\u0001\u0000\u0000\u0000\u03ea\u03e8\u0001\u0000\u0000\u0000\u03ea\u03eb"+
+    "\u0001\u0000\u0000\u0000\u03eb\u03f7\u0001\u0000\u0000\u0000\u03ec\u03ea"+
+    "\u0001\u0000\u0000\u0000\u03ed\u03f0\u0003`(\u0000\u03ee\u03f0\u0003Z"+
+    "%\u0000\u03ef\u03ed\u0001\u0000\u0000\u0000\u03ef\u03ee\u0001\u0000\u0000"+
+    "\u0000\u03f0\u03f2\u0001\u0000\u0000\u0000\u03f1\u03f3\u0003\u00dcf\u0000"+
+    "\u03f2\u03f1\u0001\u0000\u0000\u0000\u03f3\u03f4\u0001\u0000\u0000\u0000"+
+    "\u03f4\u03f2\u0001\u0000\u0000\u0000\u03f4\u03f5\u0001\u0000\u0000\u0000"+
+    "\u03f5\u03f7\u0001\u0000\u0000\u0000\u03f6\u03e5\u0001\u0000\u0000\u0000"+
+    "\u03f6\u03ef\u0001\u0000\u0000\u0000\u03f7\u00df\u0001\u0000\u0000\u0000"+
+    "\u03f8\u03fb\u0003\u00deg\u0000\u03f9\u03fb\u0003\u00b6S\u0000\u03fa\u03f8"+
+    "\u0001\u0000\u0000\u0000\u03fa\u03f9\u0001\u0000\u0000\u0000\u03fb\u03fc"+
+    "\u0001\u0000\u0000\u0000\u03fc\u03fa\u0001\u0000\u0000\u0000\u03fc\u03fd"+
+    "\u0001\u0000\u0000\u0000\u03fd\u00e1\u0001\u0000\u0000\u0000\u03fe\u03ff"+
+    "\u0003:\u0015\u0000\u03ff\u0400\u0001\u0000\u0000\u0000\u0400\u0401\u0006"+
+    "i\u000b\u0000\u0401\u00e3\u0001\u0000\u0000\u0000\u0402\u0403\u0003<\u0016"+
+    "\u0000\u0403\u0404\u0001\u0000\u0000\u0000\u0404\u0405\u0006j\u000b\u0000"+
+    "\u0405\u00e5\u0001\u0000\u0000\u0000\u0406\u0407\u0003>\u0017\u0000\u0407"+
+    "\u0408\u0001\u0000\u0000\u0000\u0408\u0409\u0006k\u000b\u0000\u0409\u00e7"+
+    "\u0001\u0000\u0000\u0000\u040a\u040b\u0003N\u001f\u0000\u040b\u040c\u0001"+
+    "\u0000\u0000\u0000\u040c\u040d\u0006l\u000e\u0000\u040d\u040e\u0006l\u000f"+
+    "\u0000\u040e\u00e9\u0001\u0000\u0000\u0000\u040f\u0410\u0003p0\u0000\u0410"+
+    "\u0411\u0001\u0000\u0000\u0000\u0411\u0412\u0006m\u0012\u0000\u0412\u00eb"+
+    "\u0001\u0000\u0000\u0000\u0413\u0414\u0003t2\u0000\u0414\u0415\u0001\u0000"+
+    "\u0000\u0000\u0415\u0416\u0006n\u0011\u0000\u0416\u00ed\u0001\u0000\u0000"+
+    "\u0000\u0417\u0418\u0003x4\u0000\u0418\u0419\u0001\u0000\u0000\u0000\u0419"+
+    "\u041a\u0006o\u0015\u0000\u041a\u00ef\u0001\u0000\u0000\u0000\u041b\u041c"+
+    "\u0005a\u0000\u0000\u041c\u041d\u0005s\u0000\u0000\u041d\u00f1\u0001\u0000"+
+    "\u0000\u0000\u041e\u041f\u0003\u00e0h\u0000\u041f\u0420\u0001\u0000\u0000"+
+    "\u0000\u0420\u0421\u0006q\u0016\u0000\u0421\u00f3\u0001\u0000\u0000\u0000"+
+    "\u0422\u0423\u0003:\u0015\u0000\u0423\u0424\u0001\u0000\u0000\u0000\u0424"+
+    "\u0425\u0006r\u000b\u0000\u0425\u00f5\u0001\u0000\u0000\u0000\u0426\u0427"+
+    "\u0003<\u0016\u0000\u0427\u0428\u0001\u0000\u0000\u0000\u0428\u0429\u0006"+
+    "s\u000b\u0000\u0429\u00f7\u0001\u0000\u0000\u0000\u042a\u042b\u0003>\u0017"+
+    "\u0000\u042b\u042c\u0001\u0000\u0000\u0000\u042c\u042d\u0006t\u000b\u0000"+
+    "\u042d\u00f9\u0001\u0000\u0000\u0000\u042e\u042f\u0003N\u001f\u0000\u042f"+
+    "\u0430\u0001\u0000\u0000\u0000\u0430\u0431\u0006u\u000e\u0000\u0431\u0432"+
+    "\u0006u\u000f\u0000\u0432\u00fb\u0001\u0000\u0000\u0000\u0433\u0434\u0003"+
+    "\u00b0P\u0000\u0434\u0435\u0001\u0000\u0000\u0000\u0435\u0436\u0006v\f"+
+    "\u0000\u0436\u0437\u0006v\u0017\u0000\u0437\u00fd\u0001\u0000\u0000\u0000"+
+    "\u0438\u0439\u0005o\u0000\u0000\u0439\u043a\u0005n\u0000\u0000\u043a\u043b"+
+    "\u0001\u0000\u0000\u0000\u043b\u043c\u0006w\u0018\u0000\u043c\u00ff\u0001"+
+    "\u0000\u0000\u0000\u043d\u043e\u0005w\u0000\u0000\u043e\u043f\u0005i\u0000"+
+    "\u0000\u043f\u0440\u0005t\u0000\u0000\u0440\u0441\u0005h\u0000\u0000\u0441"+
+    "\u0442\u0001\u0000\u0000\u0000\u0442\u0443\u0006x\u0018\u0000\u0443\u0101"+
+    "\u0001\u0000\u0000\u0000\u0444\u0445\b\f\u0000\u0000\u0445\u0103\u0001"+
+    "\u0000\u0000\u0000\u0446\u0448\u0003\u0102y\u0000\u0447\u0446\u0001\u0000"+
+    "\u0000\u0000\u0448\u0449\u0001\u0000\u0000\u0000\u0449\u0447\u0001\u0000"+
+    "\u0000\u0000\u0449\u044a\u0001\u0000\u0000\u0000\u044a\u044b\u0001\u0000"+
+    "\u0000\u0000\u044b\u044c\u0003\u0166\u00ab\u0000\u044c\u044e\u0001\u0000"+
+    "\u0000\u0000\u044d\u0447\u0001\u0000\u0000\u0000\u044d\u044e\u0001\u0000"+
+    "\u0000\u0000\u044e\u0450\u0001\u0000\u0000\u0000\u044f\u0451\u0003\u0102"+
+    "y\u0000\u0450\u044f\u0001\u0000\u0000\u0000\u0451\u0452\u0001\u0000\u0000"+
+    "\u0000\u0452\u0450\u0001\u0000\u0000\u0000\u0452\u0453\u0001\u0000\u0000"+
+    "\u0000\u0453\u0105\u0001\u0000\u0000\u0000\u0454\u0455\u0003\u00b8T\u0000"+
+    "\u0455\u0456\u0001\u0000\u0000\u0000\u0456\u0457\u0006{\u0019\u0000\u0457"+
+    "\u0107\u0001\u0000\u0000\u0000\u0458\u0459\u0003\u0104z\u0000\u0459\u045a"+
+    "\u0001\u0000\u0000\u0000\u045a\u045b\u0006|\u001a\u0000\u045b\u0109\u0001"+
+    "\u0000\u0000\u0000\u045c\u045d\u0003:\u0015\u0000\u045d\u045e\u0001\u0000"+
+    "\u0000\u0000\u045e\u045f\u0006}\u000b\u0000\u045f\u010b\u0001\u0000\u0000"+
+    "\u0000\u0460\u0461\u0003<\u0016\u0000\u0461\u0462\u0001\u0000\u0000\u0000"+
+    "\u0462\u0463\u0006~\u000b\u0000\u0463\u010d\u0001\u0000\u0000\u0000\u0464"+
+    "\u0465\u0003>\u0017\u0000\u0465\u0466\u0001\u0000\u0000\u0000\u0466\u0467"+
+    "\u0006\u007f\u000b\u0000\u0467\u010f\u0001\u0000\u0000\u0000\u0468\u0469"+
+    "\u0003N\u001f\u0000\u0469\u046a\u0001\u0000\u0000\u0000\u046a\u046b\u0006"+
+    "\u0080\u000e\u0000\u046b\u046c\u0006\u0080\u000f\u0000\u046c\u046d\u0006"+
+    "\u0080\u000f\u0000\u046d\u0111\u0001\u0000\u0000\u0000\u046e\u046f\u0003"+
+    "p0\u0000\u046f\u0470\u0001\u0000\u0000\u0000\u0470\u0471\u0006\u0081\u0012"+
+    "\u0000\u0471\u0113\u0001\u0000\u0000\u0000\u0472\u0473\u0003t2\u0000\u0473"+
+    "\u0474\u0001\u0000\u0000\u0000\u0474\u0475\u0006\u0082\u0011\u0000\u0475"+
+    "\u0115\u0001\u0000\u0000\u0000\u0476\u0477\u0003x4\u0000\u0477\u0478\u0001"+
+    "\u0000\u0000\u0000\u0478\u0479\u0006\u0083\u0015\u0000\u0479\u0117\u0001"+
+    "\u0000\u0000\u0000\u047a\u047b\u0003\u0100x\u0000\u047b\u047c\u0001\u0000"+
+    "\u0000\u0000\u047c\u047d\u0006\u0084\u001b\u0000\u047d\u0119\u0001\u0000"+
+    "\u0000\u0000\u047e\u047f\u0003\u00e0h\u0000\u047f\u0480\u0001\u0000\u0000"+
+    "\u0000\u0480\u0481\u0006\u0085\u0016\u0000\u0481\u011b\u0001\u0000\u0000"+
+    "\u0000\u0482\u0483\u0003\u00b8T\u0000\u0483\u0484\u0001\u0000\u0000\u0000"+
+    "\u0484\u0485\u0006\u0086\u0019\u0000\u0485\u011d\u0001\u0000\u0000\u0000"+
+    "\u0486\u0487\u0003:\u0015\u0000\u0487\u0488\u0001\u0000\u0000\u0000\u0488"+
+    "\u0489\u0006\u0087\u000b\u0000\u0489\u011f\u0001\u0000\u0000\u0000\u048a"+
+    "\u048b\u0003<\u0016\u0000\u048b\u048c\u0001\u0000\u0000\u0000\u048c\u048d"+
+    "\u0006\u0088\u000b\u0000\u048d\u0121\u0001\u0000\u0000\u0000\u048e\u048f"+
+    "\u0003>\u0017\u0000\u048f\u0490\u0001\u0000\u0000\u0000\u0490\u0491\u0006"+
+    "\u0089\u000b\u0000\u0491\u0123\u0001\u0000\u0000\u0000\u0492\u0493\u0003"+
+    "N\u001f\u0000\u0493\u0494\u0001\u0000\u0000\u0000\u0494\u0495\u0006\u008a"+
+    "\u000e\u0000\u0495\u0496\u0006\u008a\u000f\u0000\u0496\u0125\u0001\u0000"+
+    "\u0000\u0000\u0497\u0498\u0003t2\u0000\u0498\u0499\u0001\u0000\u0000\u0000"+
+    "\u0499\u049a\u0006\u008b\u0011\u0000\u049a\u0127\u0001\u0000\u0000\u0000"+
+    "\u049b\u049c\u0003x4\u0000\u049c\u049d\u0001\u0000\u0000\u0000\u049d\u049e"+
+    "\u0006\u008c\u0015\u0000\u049e\u0129\u0001\u0000\u0000\u0000\u049f\u04a0"+
+    "\u0003\u00few\u0000\u04a0\u04a1\u0001\u0000\u0000\u0000\u04a1\u04a2\u0006"+
+    "\u008d\u001c\u0000\u04a2\u04a3\u0006\u008d\u001d\u0000\u04a3\u012b\u0001"+
+    "\u0000\u0000\u0000\u04a4\u04a5\u0003B\u0019\u0000\u04a5\u04a6\u0001\u0000"+
+    "\u0000\u0000\u04a6\u04a7\u0006\u008e\u0014\u0000\u04a7\u012d\u0001\u0000"+
+    "\u0000\u0000\u04a8\u04a9\u0003:\u0015\u0000\u04a9\u04aa\u0001\u0000\u0000"+
+    "\u0000\u04aa\u04ab\u0006\u008f\u000b\u0000\u04ab\u012f\u0001\u0000\u0000"+
+    "\u0000\u04ac\u04ad\u0003<\u0016\u0000\u04ad\u04ae\u0001\u0000\u0000\u0000"+
+    "\u04ae\u04af\u0006\u0090\u000b\u0000\u04af\u0131\u0001\u0000\u0000\u0000"+
+    "\u04b0\u04b1\u0003>\u0017\u0000\u04b1\u04b2\u0001\u0000\u0000\u0000\u04b2"+
+    "\u04b3\u0006\u0091\u000b\u0000\u04b3\u0133\u0001\u0000\u0000\u0000\u04b4"+
+    "\u04b5\u0003N\u001f\u0000\u04b5\u04b6\u0001\u0000\u0000\u0000\u04b6\u04b7"+
+    "\u0006\u0092\u000e\u0000\u04b7\u04b8\u0006\u0092\u000f\u0000\u04b8\u04b9"+
+    "\u0006\u0092\u000f\u0000\u04b9\u0135\u0001\u0000\u0000\u0000\u04ba\u04bb"+
+    "\u0003t2\u0000\u04bb\u04bc\u0001\u0000\u0000\u0000\u04bc\u04bd\u0006\u0093"+
+    "\u0011\u0000\u04bd\u0137\u0001\u0000\u0000\u0000\u04be\u04bf\u0003x4\u0000"+
+    "\u04bf\u04c0\u0001\u0000\u0000\u0000\u04c0\u04c1\u0006\u0094\u0015\u0000"+
+    "\u04c1\u0139\u0001\u0000\u0000\u0000\u04c2\u04c3\u0003\u00e0h\u0000\u04c3"+
+    "\u04c4\u0001\u0000\u0000\u0000\u04c4\u04c5\u0006\u0095\u0016\u0000\u04c5"+
+    "\u013b\u0001\u0000\u0000\u0000\u04c6\u04c7\u0003:\u0015\u0000\u04c7\u04c8"+
+    "\u0001\u0000\u0000\u0000\u04c8\u04c9\u0006\u0096\u000b\u0000\u04c9\u013d"+
+    "\u0001\u0000\u0000\u0000\u04ca\u04cb\u0003<\u0016\u0000\u04cb\u04cc\u0001"+
+    "\u0000\u0000\u0000\u04cc\u04cd\u0006\u0097\u000b\u0000\u04cd\u013f\u0001"+
+    "\u0000\u0000\u0000\u04ce\u04cf\u0003>\u0017\u0000\u04cf\u04d0\u0001\u0000"+
+    "\u0000\u0000\u04d0\u04d1\u0006\u0098\u000b\u0000\u04d1\u0141\u0001\u0000"+
+    "\u0000\u0000\u04d2\u04d3\u0003N\u001f\u0000\u04d3\u04d4\u0001\u0000\u0000"+
+    "\u0000\u04d4\u04d5\u0006\u0099\u000e\u0000\u04d5\u04d6\u0006\u0099\u000f"+
+    "\u0000\u04d6\u0143\u0001\u0000\u0000\u0000\u04d7\u04d8\u0003x4\u0000\u04d8"+
+    "\u04d9\u0001\u0000\u0000\u0000\u04d9\u04da\u0006\u009a\u0015\u0000\u04da"+
+    "\u0145\u0001\u0000\u0000\u0000\u04db\u04dc\u0003\u00b8T\u0000\u04dc\u04dd"+
+    "\u0001\u0000\u0000\u0000\u04dd\u04de\u0006\u009b\u0019\u0000\u04de\u0147"+
+    "\u0001\u0000\u0000\u0000\u04df\u04e0\u0003\u00b4R\u0000\u04e0\u04e1\u0001"+
+    "\u0000\u0000\u0000\u04e1\u04e2\u0006\u009c\u001e\u0000\u04e2\u0149\u0001"+
+    "\u0000\u0000\u0000\u04e3\u04e4\u0003:\u0015\u0000\u04e4\u04e5\u0001\u0000"+
+    "\u0000\u0000\u04e5\u04e6\u0006\u009d\u000b\u0000\u04e6\u014b\u0001\u0000"+
+    "\u0000\u0000\u04e7\u04e8\u0003<\u0016\u0000\u04e8\u04e9\u0001\u0000\u0000"+
+    "\u0000\u04e9\u04ea\u0006\u009e\u000b\u0000\u04ea\u014d\u0001\u0000\u0000"+
+    "\u0000\u04eb\u04ec\u0003>\u0017\u0000\u04ec\u04ed\u0001\u0000\u0000\u0000"+
+    "\u04ed\u04ee\u0006\u009f\u000b\u0000\u04ee\u014f\u0001\u0000\u0000\u0000"+
+    "\u04ef\u04f0\u0003N\u001f\u0000\u04f0\u04f1\u0001\u0000\u0000\u0000\u04f1"+
+    "\u04f2\u0006\u00a0\u000e\u0000\u04f2\u04f3\u0006\u00a0\u000f\u0000\u04f3"+
+    "\u0151\u0001\u0000\u0000\u0000\u04f4\u04f5\u0005i\u0000\u0000\u04f5\u04f6"+
+    "\u0005n\u0000\u0000\u04f6\u04f7\u0005f\u0000\u0000\u04f7\u04f8\u0005o"+
+    "\u0000\u0000\u04f8\u0153\u0001\u0000\u0000\u0000\u04f9\u04fa\u0003:\u0015"+
+    "\u0000\u04fa\u04fb\u0001\u0000\u0000\u0000\u04fb\u04fc\u0006\u00a2\u000b"+
+    "\u0000\u04fc\u0155\u0001\u0000\u0000\u0000\u04fd\u04fe\u0003<\u0016\u0000"+
+    "\u04fe\u04ff\u0001\u0000\u0000\u0000\u04ff\u0500\u0006\u00a3\u000b\u0000"+
+    "\u0500\u0157\u0001\u0000\u0000\u0000\u0501\u0502\u0003>\u0017\u0000\u0502"+
+    "\u0503\u0001\u0000\u0000\u0000\u0503\u0504\u0006\u00a4\u000b\u0000\u0504"+
+    "\u0159\u0001\u0000\u0000\u0000\u0505\u0506\u0003N\u001f\u0000\u0506\u0507"+
+    "\u0001\u0000\u0000\u0000\u0507\u0508\u0006\u00a5\u000e\u0000\u0508\u0509"+
+    "\u0006\u00a5\u000f\u0000\u0509\u015b\u0001\u0000\u0000\u0000\u050a\u050b"+
+    "\u0005f\u0000\u0000\u050b\u050c\u0005u\u0000\u0000\u050c\u050d\u0005n"+
+    "\u0000\u0000\u050d\u050e\u0005c\u0000\u0000\u050e\u050f\u0005t\u0000\u0000"+
+    "\u050f\u0510\u0005i\u0000\u0000\u0510\u0511\u0005o\u0000\u0000\u0511\u0512"+
+    "\u0005n\u0000\u0000\u0512\u0513\u0005s\u0000\u0000\u0513\u015d\u0001\u0000"+
+    "\u0000\u0000\u0514\u0515\u0003:\u0015\u0000\u0515\u0516\u0001\u0000\u0000"+
+    "\u0000\u0516\u0517\u0006\u00a7\u000b\u0000\u0517\u015f\u0001\u0000\u0000"+
+    "\u0000\u0518\u0519\u0003<\u0016\u0000\u0519\u051a\u0001\u0000\u0000\u0000"+
+    "\u051a\u051b\u0006\u00a8\u000b\u0000\u051b\u0161\u0001\u0000\u0000\u0000"+
+    "\u051c\u051d\u0003>\u0017\u0000\u051d\u051e\u0001\u0000\u0000\u0000\u051e"+
+    "\u051f\u0006\u00a9\u000b\u0000\u051f\u0163\u0001\u0000\u0000\u0000\u0520"+
+    "\u0521\u0003\u00b2Q\u0000\u0521\u0522\u0001\u0000\u0000\u0000\u0522\u0523"+
+    "\u0006\u00aa\u0010\u0000\u0523\u0524\u0006\u00aa\u000f\u0000\u0524\u0165"+
+    "\u0001\u0000\u0000\u0000\u0525\u0526\u0005:\u0000\u0000\u0526\u0167\u0001"+
+    "\u0000\u0000\u0000\u0527\u052d\u0003Z%\u0000\u0528\u052d\u0003P \u0000"+
+    "\u0529\u052d\u0003x4\u0000\u052a\u052d\u0003R!\u0000\u052b\u052d\u0003"+
+    "`(\u0000\u052c\u0527\u0001\u0000\u0000\u0000\u052c\u0528\u0001\u0000\u0000"+
+    "\u0000\u052c\u0529\u0001\u0000\u0000\u0000\u052c\u052a\u0001\u0000\u0000"+
+    "\u0000\u052c\u052b\u0001\u0000\u0000\u0000\u052d\u052e\u0001\u0000\u0000"+
+    "\u0000\u052e\u052c\u0001\u0000\u0000\u0000\u052e\u052f\u0001\u0000\u0000"+
+    "\u0000\u052f\u0169\u0001\u0000\u0000\u0000\u0530\u0531\u0003:\u0015\u0000"+
+    "\u0531\u0532\u0001\u0000\u0000\u0000\u0532\u0533\u0006\u00ad\u000b\u0000"+
+    "\u0533\u016b\u0001\u0000\u0000\u0000\u0534\u0535\u0003<\u0016\u0000\u0535"+
+    "\u0536\u0001\u0000\u0000\u0000\u0536\u0537\u0006\u00ae\u000b\u0000\u0537"+
+    "\u016d\u0001\u0000\u0000\u0000\u0538\u0539\u0003>\u0017\u0000\u0539\u053a"+
+    "\u0001\u0000\u0000\u0000\u053a\u053b\u0006\u00af\u000b\u0000\u053b\u016f"+
+    "\u0001\u0000\u0000\u0000\u053c\u053d\u0003N\u001f\u0000\u053d\u053e\u0001"+
+    "\u0000\u0000\u0000\u053e\u053f\u0006\u00b0\u000e\u0000\u053f\u0540\u0006"+
+    "\u00b0\u000f\u0000\u0540\u0171\u0001\u0000\u0000\u0000\u0541\u0542\u0003"+
+    "B\u0019\u0000\u0542\u0543\u0001\u0000\u0000\u0000\u0543\u0544\u0006\u00b1"+
+    "\u0014\u0000\u0544\u0545\u0006\u00b1\u000f\u0000\u0545\u0546\u0006\u00b1"+
+    "\u001f\u0000\u0546\u0173\u0001\u0000\u0000\u0000\u0547\u0548\u0003:\u0015"+
+    "\u0000\u0548\u0549\u0001\u0000\u0000\u0000\u0549\u054a\u0006\u00b2\u000b"+
+    "\u0000\u054a\u0175\u0001\u0000\u0000\u0000\u054b\u054c\u0003<\u0016\u0000"+
+    "\u054c\u054d\u0001\u0000\u0000\u0000\u054d\u054e\u0006\u00b3\u000b\u0000"+
+    "\u054e\u0177\u0001\u0000\u0000\u0000\u054f\u0550\u0003>\u0017\u0000\u0550"+
+    "\u0551\u0001\u0000\u0000\u0000\u0551\u0552\u0006\u00b4\u000b\u0000\u0552"+
+    "\u0179\u0001\u0000\u0000\u0000\u0553\u0554\u0003t2\u0000\u0554\u0555\u0001"+
+    "\u0000\u0000\u0000\u0555\u0556\u0006\u00b5\u0011\u0000\u0556\u0557\u0006"+
+    "\u00b5\u000f\u0000\u0557\u0558\u0006\u00b5\u0007\u0000\u0558\u017b\u0001"+
+    "\u0000\u0000\u0000\u0559\u055a\u0003:\u0015\u0000\u055a\u055b\u0001\u0000"+
+    "\u0000\u0000\u055b\u055c\u0006\u00b6\u000b\u0000\u055c\u017d\u0001\u0000"+
+    "\u0000\u0000\u055d\u055e\u0003<\u0016\u0000\u055e\u055f\u0001\u0000\u0000"+
+    "\u0000\u055f\u0560\u0006\u00b7\u000b\u0000\u0560\u017f\u0001\u0000\u0000"+
+    "\u0000\u0561\u0562\u0003>\u0017\u0000\u0562\u0563\u0001\u0000\u0000\u0000"+
+    "\u0563\u0564\u0006\u00b8\u000b\u0000\u0564\u0181\u0001\u0000\u0000\u0000"+
+    "\u0565\u0566\u0003\u00b8T\u0000\u0566\u0567\u0001\u0000\u0000\u0000\u0567"+
+    "\u0568\u0006\u00b9\u000f\u0000\u0568\u0569\u0006\u00b9\u0000\u0000\u0569"+
+    "\u056a\u0006\u00b9\u0019\u0000\u056a\u0183\u0001\u0000\u0000\u0000\u056b"+
+    "\u056c\u0003\u00b4R\u0000\u056c\u056d\u0001\u0000\u0000\u0000\u056d\u056e"+
+    "\u0006\u00ba\u000f\u0000\u056e\u056f\u0006\u00ba\u0000\u0000\u056f\u0570"+
+    "\u0006\u00ba\u001e\u0000\u0570\u0185\u0001\u0000\u0000\u0000\u0571\u0572"+
+    "\u0003j-\u0000\u0572\u0573\u0001\u0000\u0000\u0000\u0573\u0574\u0006\u00bb"+
+    "\u000f\u0000\u0574\u0575\u0006\u00bb\u0000\u0000\u0575\u0576\u0006\u00bb"+
+    " \u0000\u0576\u0187\u0001\u0000\u0000\u0000\u0577\u0578\u0003N\u001f\u0000"+
+    "\u0578\u0579\u0001\u0000\u0000\u0000\u0579\u057a\u0006\u00bc\u000e\u0000"+
+    "\u057a\u057b\u0006\u00bc\u000f\u0000\u057b\u0189\u0001\u0000\u0000\u0000"+
+    ">\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e"+
+    "\u000f\u0236\u0240\u0244\u0247\u0250\u0252\u025d\u0264\u0269\u0290\u0295"+
+    "\u029e\u02a5\u02aa\u02ac\u02b7\u02bf\u02c2\u02c4\u02c9\u02ce\u02d4\u02db"+
+    "\u02e0\u02e6\u02e9\u02f1\u02f5\u037a\u037f\u0384\u0386\u038c\u03e1\u03e5"+
+    "\u03ea\u03ef\u03f4\u03f6\u03fa\u03fc\u0449\u044d\u0452\u052c\u052e!\u0005"+
+    "\u0002\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0001\u0000\u0005"+
+    "\u0003\u0000\u0005\b\u0000\u0005\f\u0000\u0005\u000e\u0000\u0005\n\u0000"+
+    "\u0005\u0005\u0000\u0005\u000b\u0000\u0000\u0001\u0000\u0007D\u0000\u0005"+
+    "\u0000\u0000\u0007\u001d\u0000\u0004\u0000\u0000\u0007E\u0000\u0007&\u0000"+
+    "\u0007$\u0000\u0007\u001e\u0000\u0007\u0019\u0000\u0007(\u0000\u0007O"+
+    "\u0000\u0005\r\u0000\u0005\u0007\u0000\u0007G\u0000\u0007Y\u0000\u0007"+
+    "X\u0000\u0007W\u0000\u0005\t\u0000\u0007F\u0000\u0005\u000f\u0000\u0007"+
+    "!\u0000";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
index d6f90975aefac..76663717c5624 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
@@ -10,6 +10,7 @@ null
 'inlinestats'
 'keep'
 'limit'
+'lookup'
 'meta'
 'metrics'
 'mv_expand'
@@ -97,6 +98,12 @@ null
 null
 null
 null
+null
+null
+null
+null
+null
+null
 'info'
 null
 null
@@ -129,6 +136,7 @@ GROK
 INLINESTATS
 KEEP
 LIMIT
+LOOKUP
 META
 METRICS
 MV_EXPAND
@@ -213,6 +221,12 @@ ENRICH_WS
 ENRICH_FIELD_LINE_COMMENT
 ENRICH_FIELD_MULTILINE_COMMENT
 ENRICH_FIELD_WS
+LOOKUP_LINE_COMMENT
+LOOKUP_MULTILINE_COMMENT
+LOOKUP_WS
+LOOKUP_FIELD_LINE_COMMENT
+LOOKUP_FIELD_MULTILINE_COMMENT
+LOOKUP_FIELD_WS
 MVEXPAND_LINE_COMMENT
 MVEXPAND_MULTILINE_COMMENT
 MVEXPAND_WS
@@ -263,6 +277,7 @@ statsCommand
 inlinestatsCommand
 qualifiedName
 qualifiedNamePattern
+qualifiedNamePatterns
 identifier
 identifierPattern
 constant
@@ -290,7 +305,8 @@ showCommand
 metaCommand
 enrichCommand
 enrichWithClause
+lookupCommand
 
 
 atn:
-[4, 1, 116, 542, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 116, 8, 1, 10, 1, 12, 1, 119, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 127, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 142, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 154, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 161, 8, 5, 10, 5, 12, 5, 164, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 171, 8, 5, 1, 5, 1, 5, 3, 5, 175, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 183, 8, 5, 10, 5, 12, 5, 186, 9, 5, 1, 6, 1, 6, 3, 6, 190, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 197, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 202, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 3, 7, 209, 8, 7, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 215, 8, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 5, 8, 223, 8, 8, 10, 8, 12, 8, 226, 9, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 236, 8, 9, 1, 9, 1, 9, 1, 9, 5, 9, 241, 8, 9, 10, 9, 12, 9, 244, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 5, 10, 252, 8, 10, 10, 10, 12, 10, 255, 9, 10, 3, 10, 257, 8, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 5, 13, 269, 8, 13, 10, 13, 12, 13, 272, 9, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 3, 14, 279, 8, 14, 1, 15, 1, 15, 1, 15, 1, 15, 5, 15, 285, 8, 15, 10, 15, 12, 15, 288, 9, 15, 1, 15, 3, 15, 291, 8, 15, 1, 16, 1, 16, 1, 17, 1, 17, 3, 17, 297, 8, 17, 1, 18, 1, 18, 1, 18, 1, 18, 5, 18, 303, 8, 18, 10, 18, 12, 18, 306, 9, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 316, 8, 20, 10, 20, 12, 20, 319, 9, 20, 1, 20, 3, 20, 322, 8, 20, 1, 20, 1, 20, 3, 20, 326, 8, 20, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 3, 22, 333, 8, 22, 1, 22, 1, 22, 3, 22, 337, 8, 22, 1, 23, 1, 23, 1, 23, 1, 23, 3, 23, 343, 8, 23, 1, 24, 1, 24, 1, 24, 5, 24, 348, 8, 24, 10, 24, 12, 24, 351, 9, 24, 1, 25, 1, 25, 1, 25, 5, 25, 356, 8, 25, 10, 25, 12, 25, 359, 9, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 5, 28, 378, 8, 28, 10, 28, 12, 28, 381, 9, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 5, 28, 389, 8, 28, 10, 28, 12, 28, 392, 9, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 5, 28, 400, 8, 28, 10, 28, 12, 28, 403, 9, 28, 1, 28, 1, 28, 3, 28, 407, 8, 28, 1, 29, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 30, 5, 30, 416, 8, 30, 10, 30, 12, 30, 419, 9, 30, 1, 31, 1, 31, 3, 31, 423, 8, 31, 1, 31, 1, 31, 3, 31, 427, 8, 31, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 433, 8, 32, 10, 32, 12, 32, 436, 9, 32, 1, 33, 1, 33, 1, 33, 1, 33, 5, 33, 442, 8, 33, 10, 33, 12, 33, 445, 9, 33, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 451, 8, 34, 10, 34, 12, 34, 454, 9, 34, 1, 35, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 1, 36, 3, 36, 464, 8, 36, 1, 37, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 5, 39, 476, 8, 39, 10, 39, 12, 39, 479, 9, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 42, 1, 42, 3, 42, 489, 8, 42, 1, 43, 3, 43, 492, 8, 43, 1, 43, 1, 43, 1, 44, 3, 44, 497, 8, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 3, 51, 522, 8, 51, 1, 51, 1, 51, 1, 51, 1, 51, 5, 51, 528, 8, 51, 10, 51, 12, 51, 531, 9, 51, 3, 51, 533, 8, 51, 1, 52, 1, 52, 1, 52, 3, 52, 538, 8, 52, 1, 52, 1, 52, 1, 52, 0, 4, 2, 10, 16, 18, 53, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 0, 7, 1, 0, 62, 63, 1, 0, 64, 66, 1, 0, 69, 70, 2, 0, 34, 34, 38, 38, 1, 0, 41, 42, 2, 0, 40, 40, 54, 54, 2, 0, 55, 55, 57, 61, 570, 0, 106, 1, 0, 0, 0, 2, 109, 1, 0, 0, 0, 4, 126, 1, 0, 0, 0, 6, 141, 1, 0, 0, 0, 8, 143, 1, 0, 0, 0, 10, 174, 1, 0, 0, 0, 12, 201, 1, 0, 0, 0, 14, 208, 1, 0, 0, 0, 16, 214, 1, 0, 0, 0, 18, 235, 1, 0, 0, 0, 20, 245, 1, 0, 0, 0, 22, 260, 1, 0, 0, 0, 24, 262, 1, 0, 0, 0, 26, 265, 1, 0, 0, 0, 28, 278, 1, 0, 0, 0, 30, 280, 1, 0, 0, 0, 32, 292, 1, 0, 0, 0, 34, 296, 1, 0, 0, 0, 36, 298, 1, 0, 0, 0, 38, 307, 1, 0, 0, 0, 40, 311, 1, 0, 0, 0, 42, 327, 1, 0, 0, 0, 44, 330, 1, 0, 0, 0, 46, 338, 1, 0, 0, 0, 48, 344, 1, 0, 0, 0, 50, 352, 1, 0, 0, 0, 52, 360, 1, 0, 0, 0, 54, 362, 1, 0, 0, 0, 56, 406, 1, 0, 0, 0, 58, 408, 1, 0, 0, 0, 60, 411, 1, 0, 0, 0, 62, 420, 1, 0, 0, 0, 64, 428, 1, 0, 0, 0, 66, 437, 1, 0, 0, 0, 68, 446, 1, 0, 0, 0, 70, 455, 1, 0, 0, 0, 72, 459, 1, 0, 0, 0, 74, 465, 1, 0, 0, 0, 76, 469, 1, 0, 0, 0, 78, 472, 1, 0, 0, 0, 80, 480, 1, 0, 0, 0, 82, 484, 1, 0, 0, 0, 84, 488, 1, 0, 0, 0, 86, 491, 1, 0, 0, 0, 88, 496, 1, 0, 0, 0, 90, 500, 1, 0, 0, 0, 92, 502, 1, 0, 0, 0, 94, 504, 1, 0, 0, 0, 96, 507, 1, 0, 0, 0, 98, 511, 1, 0, 0, 0, 100, 514, 1, 0, 0, 0, 102, 517, 1, 0, 0, 0, 104, 537, 1, 0, 0, 0, 106, 107, 3, 2, 1, 0, 107, 108, 5, 0, 0, 1, 108, 1, 1, 0, 0, 0, 109, 110, 6, 1, -1, 0, 110, 111, 3, 4, 2, 0, 111, 117, 1, 0, 0, 0, 112, 113, 10, 1, 0, 0, 113, 114, 5, 28, 0, 0, 114, 116, 3, 6, 3, 0, 115, 112, 1, 0, 0, 0, 116, 119, 1, 0, 0, 0, 117, 115, 1, 0, 0, 0, 117, 118, 1, 0, 0, 0, 118, 3, 1, 0, 0, 0, 119, 117, 1, 0, 0, 0, 120, 127, 3, 94, 47, 0, 121, 127, 3, 30, 15, 0, 122, 127, 3, 24, 12, 0, 123, 127, 3, 40, 20, 0, 124, 127, 3, 98, 49, 0, 125, 127, 3, 100, 50, 0, 126, 120, 1, 0, 0, 0, 126, 121, 1, 0, 0, 0, 126, 122, 1, 0, 0, 0, 126, 123, 1, 0, 0, 0, 126, 124, 1, 0, 0, 0, 126, 125, 1, 0, 0, 0, 127, 5, 1, 0, 0, 0, 128, 142, 3, 42, 21, 0, 129, 142, 3, 46, 23, 0, 130, 142, 3, 58, 29, 0, 131, 142, 3, 64, 32, 0, 132, 142, 3, 60, 30, 0, 133, 142, 3, 44, 22, 0, 134, 142, 3, 8, 4, 0, 135, 142, 3, 66, 33, 0, 136, 142, 3, 68, 34, 0, 137, 142, 3, 72, 36, 0, 138, 142, 3, 74, 37, 0, 139, 142, 3, 102, 51, 0, 140, 142, 3, 76, 38, 0, 141, 128, 1, 0, 0, 0, 141, 129, 1, 0, 0, 0, 141, 130, 1, 0, 0, 0, 141, 131, 1, 0, 0, 0, 141, 132, 1, 0, 0, 0, 141, 133, 1, 0, 0, 0, 141, 134, 1, 0, 0, 0, 141, 135, 1, 0, 0, 0, 141, 136, 1, 0, 0, 0, 141, 137, 1, 0, 0, 0, 141, 138, 1, 0, 0, 0, 141, 139, 1, 0, 0, 0, 141, 140, 1, 0, 0, 0, 142, 7, 1, 0, 0, 0, 143, 144, 5, 19, 0, 0, 144, 145, 3, 10, 5, 0, 145, 9, 1, 0, 0, 0, 146, 147, 6, 5, -1, 0, 147, 148, 5, 47, 0, 0, 148, 175, 3, 10, 5, 7, 149, 175, 3, 14, 7, 0, 150, 175, 3, 12, 6, 0, 151, 153, 3, 14, 7, 0, 152, 154, 5, 47, 0, 0, 153, 152, 1, 0, 0, 0, 153, 154, 1, 0, 0, 0, 154, 155, 1, 0, 0, 0, 155, 156, 5, 44, 0, 0, 156, 157, 5, 43, 0, 0, 157, 162, 3, 14, 7, 0, 158, 159, 5, 37, 0, 0, 159, 161, 3, 14, 7, 0, 160, 158, 1, 0, 0, 0, 161, 164, 1, 0, 0, 0, 162, 160, 1, 0, 0, 0, 162, 163, 1, 0, 0, 0, 163, 165, 1, 0, 0, 0, 164, 162, 1, 0, 0, 0, 165, 166, 5, 53, 0, 0, 166, 175, 1, 0, 0, 0, 167, 168, 3, 14, 7, 0, 168, 170, 5, 45, 0, 0, 169, 171, 5, 47, 0, 0, 170, 169, 1, 0, 0, 0, 170, 171, 1, 0, 0, 0, 171, 172, 1, 0, 0, 0, 172, 173, 5, 48, 0, 0, 173, 175, 1, 0, 0, 0, 174, 146, 1, 0, 0, 0, 174, 149, 1, 0, 0, 0, 174, 150, 1, 0, 0, 0, 174, 151, 1, 0, 0, 0, 174, 167, 1, 0, 0, 0, 175, 184, 1, 0, 0, 0, 176, 177, 10, 4, 0, 0, 177, 178, 5, 33, 0, 0, 178, 183, 3, 10, 5, 5, 179, 180, 10, 3, 0, 0, 180, 181, 5, 50, 0, 0, 181, 183, 3, 10, 5, 4, 182, 176, 1, 0, 0, 0, 182, 179, 1, 0, 0, 0, 183, 186, 1, 0, 0, 0, 184, 182, 1, 0, 0, 0, 184, 185, 1, 0, 0, 0, 185, 11, 1, 0, 0, 0, 186, 184, 1, 0, 0, 0, 187, 189, 3, 14, 7, 0, 188, 190, 5, 47, 0, 0, 189, 188, 1, 0, 0, 0, 189, 190, 1, 0, 0, 0, 190, 191, 1, 0, 0, 0, 191, 192, 5, 46, 0, 0, 192, 193, 3, 90, 45, 0, 193, 202, 1, 0, 0, 0, 194, 196, 3, 14, 7, 0, 195, 197, 5, 47, 0, 0, 196, 195, 1, 0, 0, 0, 196, 197, 1, 0, 0, 0, 197, 198, 1, 0, 0, 0, 198, 199, 5, 52, 0, 0, 199, 200, 3, 90, 45, 0, 200, 202, 1, 0, 0, 0, 201, 187, 1, 0, 0, 0, 201, 194, 1, 0, 0, 0, 202, 13, 1, 0, 0, 0, 203, 209, 3, 16, 8, 0, 204, 205, 3, 16, 8, 0, 205, 206, 3, 92, 46, 0, 206, 207, 3, 16, 8, 0, 207, 209, 1, 0, 0, 0, 208, 203, 1, 0, 0, 0, 208, 204, 1, 0, 0, 0, 209, 15, 1, 0, 0, 0, 210, 211, 6, 8, -1, 0, 211, 215, 3, 18, 9, 0, 212, 213, 7, 0, 0, 0, 213, 215, 3, 16, 8, 3, 214, 210, 1, 0, 0, 0, 214, 212, 1, 0, 0, 0, 215, 224, 1, 0, 0, 0, 216, 217, 10, 2, 0, 0, 217, 218, 7, 1, 0, 0, 218, 223, 3, 16, 8, 3, 219, 220, 10, 1, 0, 0, 220, 221, 7, 0, 0, 0, 221, 223, 3, 16, 8, 2, 222, 216, 1, 0, 0, 0, 222, 219, 1, 0, 0, 0, 223, 226, 1, 0, 0, 0, 224, 222, 1, 0, 0, 0, 224, 225, 1, 0, 0, 0, 225, 17, 1, 0, 0, 0, 226, 224, 1, 0, 0, 0, 227, 228, 6, 9, -1, 0, 228, 236, 3, 56, 28, 0, 229, 236, 3, 48, 24, 0, 230, 236, 3, 20, 10, 0, 231, 232, 5, 43, 0, 0, 232, 233, 3, 10, 5, 0, 233, 234, 5, 53, 0, 0, 234, 236, 1, 0, 0, 0, 235, 227, 1, 0, 0, 0, 235, 229, 1, 0, 0, 0, 235, 230, 1, 0, 0, 0, 235, 231, 1, 0, 0, 0, 236, 242, 1, 0, 0, 0, 237, 238, 10, 1, 0, 0, 238, 239, 5, 36, 0, 0, 239, 241, 3, 22, 11, 0, 240, 237, 1, 0, 0, 0, 241, 244, 1, 0, 0, 0, 242, 240, 1, 0, 0, 0, 242, 243, 1, 0, 0, 0, 243, 19, 1, 0, 0, 0, 244, 242, 1, 0, 0, 0, 245, 246, 3, 52, 26, 0, 246, 256, 5, 43, 0, 0, 247, 257, 5, 64, 0, 0, 248, 253, 3, 10, 5, 0, 249, 250, 5, 37, 0, 0, 250, 252, 3, 10, 5, 0, 251, 249, 1, 0, 0, 0, 252, 255, 1, 0, 0, 0, 253, 251, 1, 0, 0, 0, 253, 254, 1, 0, 0, 0, 254, 257, 1, 0, 0, 0, 255, 253, 1, 0, 0, 0, 256, 247, 1, 0, 0, 0, 256, 248, 1, 0, 0, 0, 256, 257, 1, 0, 0, 0, 257, 258, 1, 0, 0, 0, 258, 259, 5, 53, 0, 0, 259, 21, 1, 0, 0, 0, 260, 261, 3, 52, 26, 0, 261, 23, 1, 0, 0, 0, 262, 263, 5, 15, 0, 0, 263, 264, 3, 26, 13, 0, 264, 25, 1, 0, 0, 0, 265, 270, 3, 28, 14, 0, 266, 267, 5, 37, 0, 0, 267, 269, 3, 28, 14, 0, 268, 266, 1, 0, 0, 0, 269, 272, 1, 0, 0, 0, 270, 268, 1, 0, 0, 0, 270, 271, 1, 0, 0, 0, 271, 27, 1, 0, 0, 0, 272, 270, 1, 0, 0, 0, 273, 279, 3, 10, 5, 0, 274, 275, 3, 48, 24, 0, 275, 276, 5, 35, 0, 0, 276, 277, 3, 10, 5, 0, 277, 279, 1, 0, 0, 0, 278, 273, 1, 0, 0, 0, 278, 274, 1, 0, 0, 0, 279, 29, 1, 0, 0, 0, 280, 281, 5, 6, 0, 0, 281, 286, 3, 32, 16, 0, 282, 283, 5, 37, 0, 0, 283, 285, 3, 32, 16, 0, 284, 282, 1, 0, 0, 0, 285, 288, 1, 0, 0, 0, 286, 284, 1, 0, 0, 0, 286, 287, 1, 0, 0, 0, 287, 290, 1, 0, 0, 0, 288, 286, 1, 0, 0, 0, 289, 291, 3, 34, 17, 0, 290, 289, 1, 0, 0, 0, 290, 291, 1, 0, 0, 0, 291, 31, 1, 0, 0, 0, 292, 293, 5, 24, 0, 0, 293, 33, 1, 0, 0, 0, 294, 297, 3, 36, 18, 0, 295, 297, 3, 38, 19, 0, 296, 294, 1, 0, 0, 0, 296, 295, 1, 0, 0, 0, 297, 35, 1, 0, 0, 0, 298, 299, 5, 74, 0, 0, 299, 304, 3, 32, 16, 0, 300, 301, 5, 37, 0, 0, 301, 303, 3, 32, 16, 0, 302, 300, 1, 0, 0, 0, 303, 306, 1, 0, 0, 0, 304, 302, 1, 0, 0, 0, 304, 305, 1, 0, 0, 0, 305, 37, 1, 0, 0, 0, 306, 304, 1, 0, 0, 0, 307, 308, 5, 67, 0, 0, 308, 309, 3, 36, 18, 0, 309, 310, 5, 68, 0, 0, 310, 39, 1, 0, 0, 0, 311, 312, 5, 12, 0, 0, 312, 317, 3, 32, 16, 0, 313, 314, 5, 37, 0, 0, 314, 316, 3, 32, 16, 0, 315, 313, 1, 0, 0, 0, 316, 319, 1, 0, 0, 0, 317, 315, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 321, 1, 0, 0, 0, 319, 317, 1, 0, 0, 0, 320, 322, 3, 26, 13, 0, 321, 320, 1, 0, 0, 0, 321, 322, 1, 0, 0, 0, 322, 325, 1, 0, 0, 0, 323, 324, 5, 32, 0, 0, 324, 326, 3, 26, 13, 0, 325, 323, 1, 0, 0, 0, 325, 326, 1, 0, 0, 0, 326, 41, 1, 0, 0, 0, 327, 328, 5, 4, 0, 0, 328, 329, 3, 26, 13, 0, 329, 43, 1, 0, 0, 0, 330, 332, 5, 18, 0, 0, 331, 333, 3, 26, 13, 0, 332, 331, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 336, 1, 0, 0, 0, 334, 335, 5, 32, 0, 0, 335, 337, 3, 26, 13, 0, 336, 334, 1, 0, 0, 0, 336, 337, 1, 0, 0, 0, 337, 45, 1, 0, 0, 0, 338, 339, 5, 8, 0, 0, 339, 342, 3, 26, 13, 0, 340, 341, 5, 32, 0, 0, 341, 343, 3, 26, 13, 0, 342, 340, 1, 0, 0, 0, 342, 343, 1, 0, 0, 0, 343, 47, 1, 0, 0, 0, 344, 349, 3, 52, 26, 0, 345, 346, 5, 39, 0, 0, 346, 348, 3, 52, 26, 0, 347, 345, 1, 0, 0, 0, 348, 351, 1, 0, 0, 0, 349, 347, 1, 0, 0, 0, 349, 350, 1, 0, 0, 0, 350, 49, 1, 0, 0, 0, 351, 349, 1, 0, 0, 0, 352, 357, 3, 54, 27, 0, 353, 354, 5, 39, 0, 0, 354, 356, 3, 54, 27, 0, 355, 353, 1, 0, 0, 0, 356, 359, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 357, 358, 1, 0, 0, 0, 358, 51, 1, 0, 0, 0, 359, 357, 1, 0, 0, 0, 360, 361, 7, 2, 0, 0, 361, 53, 1, 0, 0, 0, 362, 363, 5, 78, 0, 0, 363, 55, 1, 0, 0, 0, 364, 407, 5, 48, 0, 0, 365, 366, 3, 88, 44, 0, 366, 367, 5, 69, 0, 0, 367, 407, 1, 0, 0, 0, 368, 407, 3, 86, 43, 0, 369, 407, 3, 88, 44, 0, 370, 407, 3, 82, 41, 0, 371, 407, 5, 51, 0, 0, 372, 407, 3, 90, 45, 0, 373, 374, 5, 67, 0, 0, 374, 379, 3, 84, 42, 0, 375, 376, 5, 37, 0, 0, 376, 378, 3, 84, 42, 0, 377, 375, 1, 0, 0, 0, 378, 381, 1, 0, 0, 0, 379, 377, 1, 0, 0, 0, 379, 380, 1, 0, 0, 0, 380, 382, 1, 0, 0, 0, 381, 379, 1, 0, 0, 0, 382, 383, 5, 68, 0, 0, 383, 407, 1, 0, 0, 0, 384, 385, 5, 67, 0, 0, 385, 390, 3, 82, 41, 0, 386, 387, 5, 37, 0, 0, 387, 389, 3, 82, 41, 0, 388, 386, 1, 0, 0, 0, 389, 392, 1, 0, 0, 0, 390, 388, 1, 0, 0, 0, 390, 391, 1, 0, 0, 0, 391, 393, 1, 0, 0, 0, 392, 390, 1, 0, 0, 0, 393, 394, 5, 68, 0, 0, 394, 407, 1, 0, 0, 0, 395, 396, 5, 67, 0, 0, 396, 401, 3, 90, 45, 0, 397, 398, 5, 37, 0, 0, 398, 400, 3, 90, 45, 0, 399, 397, 1, 0, 0, 0, 400, 403, 1, 0, 0, 0, 401, 399, 1, 0, 0, 0, 401, 402, 1, 0, 0, 0, 402, 404, 1, 0, 0, 0, 403, 401, 1, 0, 0, 0, 404, 405, 5, 68, 0, 0, 405, 407, 1, 0, 0, 0, 406, 364, 1, 0, 0, 0, 406, 365, 1, 0, 0, 0, 406, 368, 1, 0, 0, 0, 406, 369, 1, 0, 0, 0, 406, 370, 1, 0, 0, 0, 406, 371, 1, 0, 0, 0, 406, 372, 1, 0, 0, 0, 406, 373, 1, 0, 0, 0, 406, 384, 1, 0, 0, 0, 406, 395, 1, 0, 0, 0, 407, 57, 1, 0, 0, 0, 408, 409, 5, 10, 0, 0, 409, 410, 5, 30, 0, 0, 410, 59, 1, 0, 0, 0, 411, 412, 5, 17, 0, 0, 412, 417, 3, 62, 31, 0, 413, 414, 5, 37, 0, 0, 414, 416, 3, 62, 31, 0, 415, 413, 1, 0, 0, 0, 416, 419, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 417, 418, 1, 0, 0, 0, 418, 61, 1, 0, 0, 0, 419, 417, 1, 0, 0, 0, 420, 422, 3, 10, 5, 0, 421, 423, 7, 3, 0, 0, 422, 421, 1, 0, 0, 0, 422, 423, 1, 0, 0, 0, 423, 426, 1, 0, 0, 0, 424, 425, 5, 49, 0, 0, 425, 427, 7, 4, 0, 0, 426, 424, 1, 0, 0, 0, 426, 427, 1, 0, 0, 0, 427, 63, 1, 0, 0, 0, 428, 429, 5, 9, 0, 0, 429, 434, 3, 50, 25, 0, 430, 431, 5, 37, 0, 0, 431, 433, 3, 50, 25, 0, 432, 430, 1, 0, 0, 0, 433, 436, 1, 0, 0, 0, 434, 432, 1, 0, 0, 0, 434, 435, 1, 0, 0, 0, 435, 65, 1, 0, 0, 0, 436, 434, 1, 0, 0, 0, 437, 438, 5, 2, 0, 0, 438, 443, 3, 50, 25, 0, 439, 440, 5, 37, 0, 0, 440, 442, 3, 50, 25, 0, 441, 439, 1, 0, 0, 0, 442, 445, 1, 0, 0, 0, 443, 441, 1, 0, 0, 0, 443, 444, 1, 0, 0, 0, 444, 67, 1, 0, 0, 0, 445, 443, 1, 0, 0, 0, 446, 447, 5, 14, 0, 0, 447, 452, 3, 70, 35, 0, 448, 449, 5, 37, 0, 0, 449, 451, 3, 70, 35, 0, 450, 448, 1, 0, 0, 0, 451, 454, 1, 0, 0, 0, 452, 450, 1, 0, 0, 0, 452, 453, 1, 0, 0, 0, 453, 69, 1, 0, 0, 0, 454, 452, 1, 0, 0, 0, 455, 456, 3, 50, 25, 0, 456, 457, 5, 82, 0, 0, 457, 458, 3, 50, 25, 0, 458, 71, 1, 0, 0, 0, 459, 460, 5, 1, 0, 0, 460, 461, 3, 18, 9, 0, 461, 463, 3, 90, 45, 0, 462, 464, 3, 78, 39, 0, 463, 462, 1, 0, 0, 0, 463, 464, 1, 0, 0, 0, 464, 73, 1, 0, 0, 0, 465, 466, 5, 7, 0, 0, 466, 467, 3, 18, 9, 0, 467, 468, 3, 90, 45, 0, 468, 75, 1, 0, 0, 0, 469, 470, 5, 13, 0, 0, 470, 471, 3, 48, 24, 0, 471, 77, 1, 0, 0, 0, 472, 477, 3, 80, 40, 0, 473, 474, 5, 37, 0, 0, 474, 476, 3, 80, 40, 0, 475, 473, 1, 0, 0, 0, 476, 479, 1, 0, 0, 0, 477, 475, 1, 0, 0, 0, 477, 478, 1, 0, 0, 0, 478, 79, 1, 0, 0, 0, 479, 477, 1, 0, 0, 0, 480, 481, 3, 52, 26, 0, 481, 482, 5, 35, 0, 0, 482, 483, 3, 56, 28, 0, 483, 81, 1, 0, 0, 0, 484, 485, 7, 5, 0, 0, 485, 83, 1, 0, 0, 0, 486, 489, 3, 86, 43, 0, 487, 489, 3, 88, 44, 0, 488, 486, 1, 0, 0, 0, 488, 487, 1, 0, 0, 0, 489, 85, 1, 0, 0, 0, 490, 492, 7, 0, 0, 0, 491, 490, 1, 0, 0, 0, 491, 492, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 5, 31, 0, 0, 494, 87, 1, 0, 0, 0, 495, 497, 7, 0, 0, 0, 496, 495, 1, 0, 0, 0, 496, 497, 1, 0, 0, 0, 497, 498, 1, 0, 0, 0, 498, 499, 5, 30, 0, 0, 499, 89, 1, 0, 0, 0, 500, 501, 5, 29, 0, 0, 501, 91, 1, 0, 0, 0, 502, 503, 7, 6, 0, 0, 503, 93, 1, 0, 0, 0, 504, 505, 5, 5, 0, 0, 505, 506, 3, 96, 48, 0, 506, 95, 1, 0, 0, 0, 507, 508, 5, 67, 0, 0, 508, 509, 3, 2, 1, 0, 509, 510, 5, 68, 0, 0, 510, 97, 1, 0, 0, 0, 511, 512, 5, 16, 0, 0, 512, 513, 5, 98, 0, 0, 513, 99, 1, 0, 0, 0, 514, 515, 5, 11, 0, 0, 515, 516, 5, 102, 0, 0, 516, 101, 1, 0, 0, 0, 517, 518, 5, 3, 0, 0, 518, 521, 5, 88, 0, 0, 519, 520, 5, 86, 0, 0, 520, 522, 3, 50, 25, 0, 521, 519, 1, 0, 0, 0, 521, 522, 1, 0, 0, 0, 522, 532, 1, 0, 0, 0, 523, 524, 5, 87, 0, 0, 524, 529, 3, 104, 52, 0, 525, 526, 5, 37, 0, 0, 526, 528, 3, 104, 52, 0, 527, 525, 1, 0, 0, 0, 528, 531, 1, 0, 0, 0, 529, 527, 1, 0, 0, 0, 529, 530, 1, 0, 0, 0, 530, 533, 1, 0, 0, 0, 531, 529, 1, 0, 0, 0, 532, 523, 1, 0, 0, 0, 532, 533, 1, 0, 0, 0, 533, 103, 1, 0, 0, 0, 534, 535, 3, 50, 25, 0, 535, 536, 5, 35, 0, 0, 536, 538, 1, 0, 0, 0, 537, 534, 1, 0, 0, 0, 537, 538, 1, 0, 0, 0, 538, 539, 1, 0, 0, 0, 539, 540, 3, 50, 25, 0, 540, 105, 1, 0, 0, 0, 53, 117, 126, 141, 153, 162, 170, 174, 182, 184, 189, 196, 201, 208, 214, 222, 224, 235, 242, 253, 256, 270, 278, 286, 290, 296, 304, 317, 321, 325, 332, 336, 342, 349, 357, 379, 390, 401, 406, 417, 422, 426, 434, 443, 452, 463, 477, 488, 491, 496, 521, 529, 532, 537]
\ No newline at end of file
+[4, 1, 123, 548, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 120, 8, 1, 10, 1, 12, 1, 123, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 131, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 147, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 159, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 166, 8, 5, 10, 5, 12, 5, 169, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 176, 8, 5, 1, 5, 1, 5, 3, 5, 180, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 188, 8, 5, 10, 5, 12, 5, 191, 9, 5, 1, 6, 1, 6, 3, 6, 195, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 202, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 207, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 3, 7, 214, 8, 7, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 220, 8, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 5, 8, 228, 8, 8, 10, 8, 12, 8, 231, 9, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 241, 8, 9, 1, 9, 1, 9, 1, 9, 5, 9, 246, 8, 9, 10, 9, 12, 9, 249, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 5, 10, 257, 8, 10, 10, 10, 12, 10, 260, 9, 10, 3, 10, 262, 8, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 5, 13, 274, 8, 13, 10, 13, 12, 13, 277, 9, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 3, 14, 284, 8, 14, 1, 15, 1, 15, 1, 15, 1, 15, 5, 15, 290, 8, 15, 10, 15, 12, 15, 293, 9, 15, 1, 15, 3, 15, 296, 8, 15, 1, 16, 1, 16, 1, 17, 1, 17, 3, 17, 302, 8, 17, 1, 18, 1, 18, 1, 18, 1, 18, 5, 18, 308, 8, 18, 10, 18, 12, 18, 311, 9, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 5, 20, 321, 8, 20, 10, 20, 12, 20, 324, 9, 20, 1, 20, 3, 20, 327, 8, 20, 1, 20, 1, 20, 3, 20, 331, 8, 20, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 3, 22, 338, 8, 22, 1, 22, 1, 22, 3, 22, 342, 8, 22, 1, 23, 1, 23, 1, 23, 1, 23, 3, 23, 348, 8, 23, 1, 24, 1, 24, 1, 24, 5, 24, 353, 8, 24, 10, 24, 12, 24, 356, 9, 24, 1, 25, 1, 25, 1, 25, 5, 25, 361, 8, 25, 10, 25, 12, 25, 364, 9, 25, 1, 26, 1, 26, 1, 26, 5, 26, 369, 8, 26, 10, 26, 12, 26, 372, 9, 26, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 5, 29, 391, 8, 29, 10, 29, 12, 29, 394, 9, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 5, 29, 402, 8, 29, 10, 29, 12, 29, 405, 9, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 1, 29, 5, 29, 413, 8, 29, 10, 29, 12, 29, 416, 9, 29, 1, 29, 1, 29, 3, 29, 420, 8, 29, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 429, 8, 31, 10, 31, 12, 31, 432, 9, 31, 1, 32, 1, 32, 3, 32, 436, 8, 32, 1, 32, 1, 32, 3, 32, 440, 8, 32, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 452, 8, 35, 10, 35, 12, 35, 455, 9, 35, 1, 36, 1, 36, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 1, 37, 3, 37, 465, 8, 37, 1, 38, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 5, 40, 477, 8, 40, 10, 40, 12, 40, 480, 9, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 3, 43, 490, 8, 43, 1, 44, 3, 44, 493, 8, 44, 1, 44, 1, 44, 1, 45, 3, 45, 498, 8, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 3, 52, 523, 8, 52, 1, 52, 1, 52, 1, 52, 1, 52, 5, 52, 529, 8, 52, 10, 52, 12, 52, 532, 9, 52, 3, 52, 534, 8, 52, 1, 53, 1, 53, 1, 53, 3, 53, 539, 8, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 0, 4, 2, 10, 16, 18, 55, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 0, 7, 1, 0, 63, 64, 1, 0, 65, 67, 1, 0, 70, 71, 2, 0, 35, 35, 39, 39, 1, 0, 42, 43, 2, 0, 41, 41, 55, 55, 2, 0, 56, 56, 58, 62, 574, 0, 110, 1, 0, 0, 0, 2, 113, 1, 0, 0, 0, 4, 130, 1, 0, 0, 0, 6, 146, 1, 0, 0, 0, 8, 148, 1, 0, 0, 0, 10, 179, 1, 0, 0, 0, 12, 206, 1, 0, 0, 0, 14, 213, 1, 0, 0, 0, 16, 219, 1, 0, 0, 0, 18, 240, 1, 0, 0, 0, 20, 250, 1, 0, 0, 0, 22, 265, 1, 0, 0, 0, 24, 267, 1, 0, 0, 0, 26, 270, 1, 0, 0, 0, 28, 283, 1, 0, 0, 0, 30, 285, 1, 0, 0, 0, 32, 297, 1, 0, 0, 0, 34, 301, 1, 0, 0, 0, 36, 303, 1, 0, 0, 0, 38, 312, 1, 0, 0, 0, 40, 316, 1, 0, 0, 0, 42, 332, 1, 0, 0, 0, 44, 335, 1, 0, 0, 0, 46, 343, 1, 0, 0, 0, 48, 349, 1, 0, 0, 0, 50, 357, 1, 0, 0, 0, 52, 365, 1, 0, 0, 0, 54, 373, 1, 0, 0, 0, 56, 375, 1, 0, 0, 0, 58, 419, 1, 0, 0, 0, 60, 421, 1, 0, 0, 0, 62, 424, 1, 0, 0, 0, 64, 433, 1, 0, 0, 0, 66, 441, 1, 0, 0, 0, 68, 444, 1, 0, 0, 0, 70, 447, 1, 0, 0, 0, 72, 456, 1, 0, 0, 0, 74, 460, 1, 0, 0, 0, 76, 466, 1, 0, 0, 0, 78, 470, 1, 0, 0, 0, 80, 473, 1, 0, 0, 0, 82, 481, 1, 0, 0, 0, 84, 485, 1, 0, 0, 0, 86, 489, 1, 0, 0, 0, 88, 492, 1, 0, 0, 0, 90, 497, 1, 0, 0, 0, 92, 501, 1, 0, 0, 0, 94, 503, 1, 0, 0, 0, 96, 505, 1, 0, 0, 0, 98, 508, 1, 0, 0, 0, 100, 512, 1, 0, 0, 0, 102, 515, 1, 0, 0, 0, 104, 518, 1, 0, 0, 0, 106, 538, 1, 0, 0, 0, 108, 542, 1, 0, 0, 0, 110, 111, 3, 2, 1, 0, 111, 112, 5, 0, 0, 1, 112, 1, 1, 0, 0, 0, 113, 114, 6, 1, -1, 0, 114, 115, 3, 4, 2, 0, 115, 121, 1, 0, 0, 0, 116, 117, 10, 1, 0, 0, 117, 118, 5, 29, 0, 0, 118, 120, 3, 6, 3, 0, 119, 116, 1, 0, 0, 0, 120, 123, 1, 0, 0, 0, 121, 119, 1, 0, 0, 0, 121, 122, 1, 0, 0, 0, 122, 3, 1, 0, 0, 0, 123, 121, 1, 0, 0, 0, 124, 131, 3, 96, 48, 0, 125, 131, 3, 30, 15, 0, 126, 131, 3, 24, 12, 0, 127, 131, 3, 40, 20, 0, 128, 131, 3, 100, 50, 0, 129, 131, 3, 102, 51, 0, 130, 124, 1, 0, 0, 0, 130, 125, 1, 0, 0, 0, 130, 126, 1, 0, 0, 0, 130, 127, 1, 0, 0, 0, 130, 128, 1, 0, 0, 0, 130, 129, 1, 0, 0, 0, 131, 5, 1, 0, 0, 0, 132, 147, 3, 42, 21, 0, 133, 147, 3, 46, 23, 0, 134, 147, 3, 60, 30, 0, 135, 147, 3, 108, 54, 0, 136, 147, 3, 66, 33, 0, 137, 147, 3, 62, 31, 0, 138, 147, 3, 44, 22, 0, 139, 147, 3, 8, 4, 0, 140, 147, 3, 68, 34, 0, 141, 147, 3, 70, 35, 0, 142, 147, 3, 74, 37, 0, 143, 147, 3, 76, 38, 0, 144, 147, 3, 104, 52, 0, 145, 147, 3, 78, 39, 0, 146, 132, 1, 0, 0, 0, 146, 133, 1, 0, 0, 0, 146, 134, 1, 0, 0, 0, 146, 135, 1, 0, 0, 0, 146, 136, 1, 0, 0, 0, 146, 137, 1, 0, 0, 0, 146, 138, 1, 0, 0, 0, 146, 139, 1, 0, 0, 0, 146, 140, 1, 0, 0, 0, 146, 141, 1, 0, 0, 0, 146, 142, 1, 0, 0, 0, 146, 143, 1, 0, 0, 0, 146, 144, 1, 0, 0, 0, 146, 145, 1, 0, 0, 0, 147, 7, 1, 0, 0, 0, 148, 149, 5, 20, 0, 0, 149, 150, 3, 10, 5, 0, 150, 9, 1, 0, 0, 0, 151, 152, 6, 5, -1, 0, 152, 153, 5, 48, 0, 0, 153, 180, 3, 10, 5, 7, 154, 180, 3, 14, 7, 0, 155, 180, 3, 12, 6, 0, 156, 158, 3, 14, 7, 0, 157, 159, 5, 48, 0, 0, 158, 157, 1, 0, 0, 0, 158, 159, 1, 0, 0, 0, 159, 160, 1, 0, 0, 0, 160, 161, 5, 45, 0, 0, 161, 162, 5, 44, 0, 0, 162, 167, 3, 14, 7, 0, 163, 164, 5, 38, 0, 0, 164, 166, 3, 14, 7, 0, 165, 163, 1, 0, 0, 0, 166, 169, 1, 0, 0, 0, 167, 165, 1, 0, 0, 0, 167, 168, 1, 0, 0, 0, 168, 170, 1, 0, 0, 0, 169, 167, 1, 0, 0, 0, 170, 171, 5, 54, 0, 0, 171, 180, 1, 0, 0, 0, 172, 173, 3, 14, 7, 0, 173, 175, 5, 46, 0, 0, 174, 176, 5, 48, 0, 0, 175, 174, 1, 0, 0, 0, 175, 176, 1, 0, 0, 0, 176, 177, 1, 0, 0, 0, 177, 178, 5, 49, 0, 0, 178, 180, 1, 0, 0, 0, 179, 151, 1, 0, 0, 0, 179, 154, 1, 0, 0, 0, 179, 155, 1, 0, 0, 0, 179, 156, 1, 0, 0, 0, 179, 172, 1, 0, 0, 0, 180, 189, 1, 0, 0, 0, 181, 182, 10, 4, 0, 0, 182, 183, 5, 34, 0, 0, 183, 188, 3, 10, 5, 5, 184, 185, 10, 3, 0, 0, 185, 186, 5, 51, 0, 0, 186, 188, 3, 10, 5, 4, 187, 181, 1, 0, 0, 0, 187, 184, 1, 0, 0, 0, 188, 191, 1, 0, 0, 0, 189, 187, 1, 0, 0, 0, 189, 190, 1, 0, 0, 0, 190, 11, 1, 0, 0, 0, 191, 189, 1, 0, 0, 0, 192, 194, 3, 14, 7, 0, 193, 195, 5, 48, 0, 0, 194, 193, 1, 0, 0, 0, 194, 195, 1, 0, 0, 0, 195, 196, 1, 0, 0, 0, 196, 197, 5, 47, 0, 0, 197, 198, 3, 92, 46, 0, 198, 207, 1, 0, 0, 0, 199, 201, 3, 14, 7, 0, 200, 202, 5, 48, 0, 0, 201, 200, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 203, 1, 0, 0, 0, 203, 204, 5, 53, 0, 0, 204, 205, 3, 92, 46, 0, 205, 207, 1, 0, 0, 0, 206, 192, 1, 0, 0, 0, 206, 199, 1, 0, 0, 0, 207, 13, 1, 0, 0, 0, 208, 214, 3, 16, 8, 0, 209, 210, 3, 16, 8, 0, 210, 211, 3, 94, 47, 0, 211, 212, 3, 16, 8, 0, 212, 214, 1, 0, 0, 0, 213, 208, 1, 0, 0, 0, 213, 209, 1, 0, 0, 0, 214, 15, 1, 0, 0, 0, 215, 216, 6, 8, -1, 0, 216, 220, 3, 18, 9, 0, 217, 218, 7, 0, 0, 0, 218, 220, 3, 16, 8, 3, 219, 215, 1, 0, 0, 0, 219, 217, 1, 0, 0, 0, 220, 229, 1, 0, 0, 0, 221, 222, 10, 2, 0, 0, 222, 223, 7, 1, 0, 0, 223, 228, 3, 16, 8, 3, 224, 225, 10, 1, 0, 0, 225, 226, 7, 0, 0, 0, 226, 228, 3, 16, 8, 2, 227, 221, 1, 0, 0, 0, 227, 224, 1, 0, 0, 0, 228, 231, 1, 0, 0, 0, 229, 227, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 17, 1, 0, 0, 0, 231, 229, 1, 0, 0, 0, 232, 233, 6, 9, -1, 0, 233, 241, 3, 58, 29, 0, 234, 241, 3, 48, 24, 0, 235, 241, 3, 20, 10, 0, 236, 237, 5, 44, 0, 0, 237, 238, 3, 10, 5, 0, 238, 239, 5, 54, 0, 0, 239, 241, 1, 0, 0, 0, 240, 232, 1, 0, 0, 0, 240, 234, 1, 0, 0, 0, 240, 235, 1, 0, 0, 0, 240, 236, 1, 0, 0, 0, 241, 247, 1, 0, 0, 0, 242, 243, 10, 1, 0, 0, 243, 244, 5, 37, 0, 0, 244, 246, 3, 22, 11, 0, 245, 242, 1, 0, 0, 0, 246, 249, 1, 0, 0, 0, 247, 245, 1, 0, 0, 0, 247, 248, 1, 0, 0, 0, 248, 19, 1, 0, 0, 0, 249, 247, 1, 0, 0, 0, 250, 251, 3, 54, 27, 0, 251, 261, 5, 44, 0, 0, 252, 262, 5, 65, 0, 0, 253, 258, 3, 10, 5, 0, 254, 255, 5, 38, 0, 0, 255, 257, 3, 10, 5, 0, 256, 254, 1, 0, 0, 0, 257, 260, 1, 0, 0, 0, 258, 256, 1, 0, 0, 0, 258, 259, 1, 0, 0, 0, 259, 262, 1, 0, 0, 0, 260, 258, 1, 0, 0, 0, 261, 252, 1, 0, 0, 0, 261, 253, 1, 0, 0, 0, 261, 262, 1, 0, 0, 0, 262, 263, 1, 0, 0, 0, 263, 264, 5, 54, 0, 0, 264, 21, 1, 0, 0, 0, 265, 266, 3, 54, 27, 0, 266, 23, 1, 0, 0, 0, 267, 268, 5, 16, 0, 0, 268, 269, 3, 26, 13, 0, 269, 25, 1, 0, 0, 0, 270, 275, 3, 28, 14, 0, 271, 272, 5, 38, 0, 0, 272, 274, 3, 28, 14, 0, 273, 271, 1, 0, 0, 0, 274, 277, 1, 0, 0, 0, 275, 273, 1, 0, 0, 0, 275, 276, 1, 0, 0, 0, 276, 27, 1, 0, 0, 0, 277, 275, 1, 0, 0, 0, 278, 284, 3, 10, 5, 0, 279, 280, 3, 48, 24, 0, 280, 281, 5, 36, 0, 0, 281, 282, 3, 10, 5, 0, 282, 284, 1, 0, 0, 0, 283, 278, 1, 0, 0, 0, 283, 279, 1, 0, 0, 0, 284, 29, 1, 0, 0, 0, 285, 286, 5, 6, 0, 0, 286, 291, 3, 32, 16, 0, 287, 288, 5, 38, 0, 0, 288, 290, 3, 32, 16, 0, 289, 287, 1, 0, 0, 0, 290, 293, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 291, 292, 1, 0, 0, 0, 292, 295, 1, 0, 0, 0, 293, 291, 1, 0, 0, 0, 294, 296, 3, 34, 17, 0, 295, 294, 1, 0, 0, 0, 295, 296, 1, 0, 0, 0, 296, 31, 1, 0, 0, 0, 297, 298, 5, 25, 0, 0, 298, 33, 1, 0, 0, 0, 299, 302, 3, 36, 18, 0, 300, 302, 3, 38, 19, 0, 301, 299, 1, 0, 0, 0, 301, 300, 1, 0, 0, 0, 302, 35, 1, 0, 0, 0, 303, 304, 5, 75, 0, 0, 304, 309, 3, 32, 16, 0, 305, 306, 5, 38, 0, 0, 306, 308, 3, 32, 16, 0, 307, 305, 1, 0, 0, 0, 308, 311, 1, 0, 0, 0, 309, 307, 1, 0, 0, 0, 309, 310, 1, 0, 0, 0, 310, 37, 1, 0, 0, 0, 311, 309, 1, 0, 0, 0, 312, 313, 5, 68, 0, 0, 313, 314, 3, 36, 18, 0, 314, 315, 5, 69, 0, 0, 315, 39, 1, 0, 0, 0, 316, 317, 5, 13, 0, 0, 317, 322, 3, 32, 16, 0, 318, 319, 5, 38, 0, 0, 319, 321, 3, 32, 16, 0, 320, 318, 1, 0, 0, 0, 321, 324, 1, 0, 0, 0, 322, 320, 1, 0, 0, 0, 322, 323, 1, 0, 0, 0, 323, 326, 1, 0, 0, 0, 324, 322, 1, 0, 0, 0, 325, 327, 3, 26, 13, 0, 326, 325, 1, 0, 0, 0, 326, 327, 1, 0, 0, 0, 327, 330, 1, 0, 0, 0, 328, 329, 5, 33, 0, 0, 329, 331, 3, 26, 13, 0, 330, 328, 1, 0, 0, 0, 330, 331, 1, 0, 0, 0, 331, 41, 1, 0, 0, 0, 332, 333, 5, 4, 0, 0, 333, 334, 3, 26, 13, 0, 334, 43, 1, 0, 0, 0, 335, 337, 5, 19, 0, 0, 336, 338, 3, 26, 13, 0, 337, 336, 1, 0, 0, 0, 337, 338, 1, 0, 0, 0, 338, 341, 1, 0, 0, 0, 339, 340, 5, 33, 0, 0, 340, 342, 3, 26, 13, 0, 341, 339, 1, 0, 0, 0, 341, 342, 1, 0, 0, 0, 342, 45, 1, 0, 0, 0, 343, 344, 5, 8, 0, 0, 344, 347, 3, 26, 13, 0, 345, 346, 5, 33, 0, 0, 346, 348, 3, 26, 13, 0, 347, 345, 1, 0, 0, 0, 347, 348, 1, 0, 0, 0, 348, 47, 1, 0, 0, 0, 349, 354, 3, 54, 27, 0, 350, 351, 5, 40, 0, 0, 351, 353, 3, 54, 27, 0, 352, 350, 1, 0, 0, 0, 353, 356, 1, 0, 0, 0, 354, 352, 1, 0, 0, 0, 354, 355, 1, 0, 0, 0, 355, 49, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 357, 362, 3, 56, 28, 0, 358, 359, 5, 40, 0, 0, 359, 361, 3, 56, 28, 0, 360, 358, 1, 0, 0, 0, 361, 364, 1, 0, 0, 0, 362, 360, 1, 0, 0, 0, 362, 363, 1, 0, 0, 0, 363, 51, 1, 0, 0, 0, 364, 362, 1, 0, 0, 0, 365, 370, 3, 50, 25, 0, 366, 367, 5, 38, 0, 0, 367, 369, 3, 50, 25, 0, 368, 366, 1, 0, 0, 0, 369, 372, 1, 0, 0, 0, 370, 368, 1, 0, 0, 0, 370, 371, 1, 0, 0, 0, 371, 53, 1, 0, 0, 0, 372, 370, 1, 0, 0, 0, 373, 374, 7, 2, 0, 0, 374, 55, 1, 0, 0, 0, 375, 376, 5, 79, 0, 0, 376, 57, 1, 0, 0, 0, 377, 420, 5, 49, 0, 0, 378, 379, 3, 90, 45, 0, 379, 380, 5, 70, 0, 0, 380, 420, 1, 0, 0, 0, 381, 420, 3, 88, 44, 0, 382, 420, 3, 90, 45, 0, 383, 420, 3, 84, 42, 0, 384, 420, 5, 52, 0, 0, 385, 420, 3, 92, 46, 0, 386, 387, 5, 68, 0, 0, 387, 392, 3, 86, 43, 0, 388, 389, 5, 38, 0, 0, 389, 391, 3, 86, 43, 0, 390, 388, 1, 0, 0, 0, 391, 394, 1, 0, 0, 0, 392, 390, 1, 0, 0, 0, 392, 393, 1, 0, 0, 0, 393, 395, 1, 0, 0, 0, 394, 392, 1, 0, 0, 0, 395, 396, 5, 69, 0, 0, 396, 420, 1, 0, 0, 0, 397, 398, 5, 68, 0, 0, 398, 403, 3, 84, 42, 0, 399, 400, 5, 38, 0, 0, 400, 402, 3, 84, 42, 0, 401, 399, 1, 0, 0, 0, 402, 405, 1, 0, 0, 0, 403, 401, 1, 0, 0, 0, 403, 404, 1, 0, 0, 0, 404, 406, 1, 0, 0, 0, 405, 403, 1, 0, 0, 0, 406, 407, 5, 69, 0, 0, 407, 420, 1, 0, 0, 0, 408, 409, 5, 68, 0, 0, 409, 414, 3, 92, 46, 0, 410, 411, 5, 38, 0, 0, 411, 413, 3, 92, 46, 0, 412, 410, 1, 0, 0, 0, 413, 416, 1, 0, 0, 0, 414, 412, 1, 0, 0, 0, 414, 415, 1, 0, 0, 0, 415, 417, 1, 0, 0, 0, 416, 414, 1, 0, 0, 0, 417, 418, 5, 69, 0, 0, 418, 420, 1, 0, 0, 0, 419, 377, 1, 0, 0, 0, 419, 378, 1, 0, 0, 0, 419, 381, 1, 0, 0, 0, 419, 382, 1, 0, 0, 0, 419, 383, 1, 0, 0, 0, 419, 384, 1, 0, 0, 0, 419, 385, 1, 0, 0, 0, 419, 386, 1, 0, 0, 0, 419, 397, 1, 0, 0, 0, 419, 408, 1, 0, 0, 0, 420, 59, 1, 0, 0, 0, 421, 422, 5, 10, 0, 0, 422, 423, 5, 31, 0, 0, 423, 61, 1, 0, 0, 0, 424, 425, 5, 18, 0, 0, 425, 430, 3, 64, 32, 0, 426, 427, 5, 38, 0, 0, 427, 429, 3, 64, 32, 0, 428, 426, 1, 0, 0, 0, 429, 432, 1, 0, 0, 0, 430, 428, 1, 0, 0, 0, 430, 431, 1, 0, 0, 0, 431, 63, 1, 0, 0, 0, 432, 430, 1, 0, 0, 0, 433, 435, 3, 10, 5, 0, 434, 436, 7, 3, 0, 0, 435, 434, 1, 0, 0, 0, 435, 436, 1, 0, 0, 0, 436, 439, 1, 0, 0, 0, 437, 438, 5, 50, 0, 0, 438, 440, 7, 4, 0, 0, 439, 437, 1, 0, 0, 0, 439, 440, 1, 0, 0, 0, 440, 65, 1, 0, 0, 0, 441, 442, 5, 9, 0, 0, 442, 443, 3, 52, 26, 0, 443, 67, 1, 0, 0, 0, 444, 445, 5, 2, 0, 0, 445, 446, 3, 52, 26, 0, 446, 69, 1, 0, 0, 0, 447, 448, 5, 15, 0, 0, 448, 453, 3, 72, 36, 0, 449, 450, 5, 38, 0, 0, 450, 452, 3, 72, 36, 0, 451, 449, 1, 0, 0, 0, 452, 455, 1, 0, 0, 0, 453, 451, 1, 0, 0, 0, 453, 454, 1, 0, 0, 0, 454, 71, 1, 0, 0, 0, 455, 453, 1, 0, 0, 0, 456, 457, 3, 50, 25, 0, 457, 458, 5, 83, 0, 0, 458, 459, 3, 50, 25, 0, 459, 73, 1, 0, 0, 0, 460, 461, 5, 1, 0, 0, 461, 462, 3, 18, 9, 0, 462, 464, 3, 92, 46, 0, 463, 465, 3, 80, 40, 0, 464, 463, 1, 0, 0, 0, 464, 465, 1, 0, 0, 0, 465, 75, 1, 0, 0, 0, 466, 467, 5, 7, 0, 0, 467, 468, 3, 18, 9, 0, 468, 469, 3, 92, 46, 0, 469, 77, 1, 0, 0, 0, 470, 471, 5, 14, 0, 0, 471, 472, 3, 48, 24, 0, 472, 79, 1, 0, 0, 0, 473, 478, 3, 82, 41, 0, 474, 475, 5, 38, 0, 0, 475, 477, 3, 82, 41, 0, 476, 474, 1, 0, 0, 0, 477, 480, 1, 0, 0, 0, 478, 476, 1, 0, 0, 0, 478, 479, 1, 0, 0, 0, 479, 81, 1, 0, 0, 0, 480, 478, 1, 0, 0, 0, 481, 482, 3, 54, 27, 0, 482, 483, 5, 36, 0, 0, 483, 484, 3, 58, 29, 0, 484, 83, 1, 0, 0, 0, 485, 486, 7, 5, 0, 0, 486, 85, 1, 0, 0, 0, 487, 490, 3, 88, 44, 0, 488, 490, 3, 90, 45, 0, 489, 487, 1, 0, 0, 0, 489, 488, 1, 0, 0, 0, 490, 87, 1, 0, 0, 0, 491, 493, 7, 0, 0, 0, 492, 491, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 494, 1, 0, 0, 0, 494, 495, 5, 32, 0, 0, 495, 89, 1, 0, 0, 0, 496, 498, 7, 0, 0, 0, 497, 496, 1, 0, 0, 0, 497, 498, 1, 0, 0, 0, 498, 499, 1, 0, 0, 0, 499, 500, 5, 31, 0, 0, 500, 91, 1, 0, 0, 0, 501, 502, 5, 30, 0, 0, 502, 93, 1, 0, 0, 0, 503, 504, 7, 6, 0, 0, 504, 95, 1, 0, 0, 0, 505, 506, 5, 5, 0, 0, 506, 507, 3, 98, 49, 0, 507, 97, 1, 0, 0, 0, 508, 509, 5, 68, 0, 0, 509, 510, 3, 2, 1, 0, 510, 511, 5, 69, 0, 0, 511, 99, 1, 0, 0, 0, 512, 513, 5, 17, 0, 0, 513, 514, 5, 105, 0, 0, 514, 101, 1, 0, 0, 0, 515, 516, 5, 12, 0, 0, 516, 517, 5, 109, 0, 0, 517, 103, 1, 0, 0, 0, 518, 519, 5, 3, 0, 0, 519, 522, 5, 89, 0, 0, 520, 521, 5, 87, 0, 0, 521, 523, 3, 50, 25, 0, 522, 520, 1, 0, 0, 0, 522, 523, 1, 0, 0, 0, 523, 533, 1, 0, 0, 0, 524, 525, 5, 88, 0, 0, 525, 530, 3, 106, 53, 0, 526, 527, 5, 38, 0, 0, 527, 529, 3, 106, 53, 0, 528, 526, 1, 0, 0, 0, 529, 532, 1, 0, 0, 0, 530, 528, 1, 0, 0, 0, 530, 531, 1, 0, 0, 0, 531, 534, 1, 0, 0, 0, 532, 530, 1, 0, 0, 0, 533, 524, 1, 0, 0, 0, 533, 534, 1, 0, 0, 0, 534, 105, 1, 0, 0, 0, 535, 536, 3, 50, 25, 0, 536, 537, 5, 36, 0, 0, 537, 539, 1, 0, 0, 0, 538, 535, 1, 0, 0, 0, 538, 539, 1, 0, 0, 0, 539, 540, 1, 0, 0, 0, 540, 541, 3, 50, 25, 0, 541, 107, 1, 0, 0, 0, 542, 543, 5, 11, 0, 0, 543, 544, 5, 25, 0, 0, 544, 545, 5, 87, 0, 0, 545, 546, 3, 52, 26, 0, 546, 109, 1, 0, 0, 0, 52, 121, 130, 146, 158, 167, 175, 179, 187, 189, 194, 201, 206, 213, 219, 227, 229, 240, 247, 258, 261, 275, 283, 291, 295, 301, 309, 322, 326, 330, 337, 341, 347, 354, 362, 370, 392, 403, 414, 419, 430, 435, 439, 453, 464, 478, 489, 492, 497, 522, 530, 533, 538]
\ No newline at end of file
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
index 1d6be711f9814..1c136a946a0f3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
@@ -18,28 +18,30 @@ public class EsqlBaseParser extends Parser {
     new PredictionContextCache();
   public static final int
     DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, INLINESTATS=8, 
-    KEEP=9, LIMIT=10, META=11, METRICS=12, MV_EXPAND=13, RENAME=14, ROW=15, 
-    SHOW=16, SORT=17, STATS=18, WHERE=19, UNKNOWN_CMD=20, LINE_COMMENT=21, 
-    MULTILINE_COMMENT=22, WS=23, INDEX_UNQUOTED_IDENTIFIER=24, EXPLAIN_WS=25, 
-    EXPLAIN_LINE_COMMENT=26, EXPLAIN_MULTILINE_COMMENT=27, PIPE=28, QUOTED_STRING=29, 
-    INTEGER_LITERAL=30, DECIMAL_LITERAL=31, BY=32, AND=33, ASC=34, ASSIGN=35, 
-    CAST_OP=36, COMMA=37, DESC=38, DOT=39, FALSE=40, FIRST=41, LAST=42, LP=43, 
-    IN=44, IS=45, LIKE=46, NOT=47, NULL=48, NULLS=49, OR=50, PARAM=51, RLIKE=52, 
-    RP=53, TRUE=54, EQ=55, CIEQ=56, NEQ=57, LT=58, LTE=59, GT=60, GTE=61, 
-    PLUS=62, MINUS=63, ASTERISK=64, SLASH=65, PERCENT=66, OPENING_BRACKET=67, 
-    CLOSING_BRACKET=68, UNQUOTED_IDENTIFIER=69, QUOTED_IDENTIFIER=70, EXPR_LINE_COMMENT=71, 
-    EXPR_MULTILINE_COMMENT=72, EXPR_WS=73, METADATA=74, FROM_LINE_COMMENT=75, 
-    FROM_MULTILINE_COMMENT=76, FROM_WS=77, ID_PATTERN=78, PROJECT_LINE_COMMENT=79, 
-    PROJECT_MULTILINE_COMMENT=80, PROJECT_WS=81, AS=82, RENAME_LINE_COMMENT=83, 
-    RENAME_MULTILINE_COMMENT=84, RENAME_WS=85, ON=86, WITH=87, ENRICH_POLICY_NAME=88, 
-    ENRICH_LINE_COMMENT=89, ENRICH_MULTILINE_COMMENT=90, ENRICH_WS=91, ENRICH_FIELD_LINE_COMMENT=92, 
-    ENRICH_FIELD_MULTILINE_COMMENT=93, ENRICH_FIELD_WS=94, MVEXPAND_LINE_COMMENT=95, 
-    MVEXPAND_MULTILINE_COMMENT=96, MVEXPAND_WS=97, INFO=98, SHOW_LINE_COMMENT=99, 
-    SHOW_MULTILINE_COMMENT=100, SHOW_WS=101, FUNCTIONS=102, META_LINE_COMMENT=103, 
-    META_MULTILINE_COMMENT=104, META_WS=105, COLON=106, SETTING=107, SETTING_LINE_COMMENT=108, 
-    SETTTING_MULTILINE_COMMENT=109, SETTING_WS=110, METRICS_LINE_COMMENT=111, 
-    METRICS_MULTILINE_COMMENT=112, METRICS_WS=113, CLOSING_METRICS_LINE_COMMENT=114, 
-    CLOSING_METRICS_MULTILINE_COMMENT=115, CLOSING_METRICS_WS=116;
+    KEEP=9, LIMIT=10, LOOKUP=11, META=12, METRICS=13, MV_EXPAND=14, RENAME=15, 
+    ROW=16, SHOW=17, SORT=18, STATS=19, WHERE=20, UNKNOWN_CMD=21, LINE_COMMENT=22, 
+    MULTILINE_COMMENT=23, WS=24, INDEX_UNQUOTED_IDENTIFIER=25, EXPLAIN_WS=26, 
+    EXPLAIN_LINE_COMMENT=27, EXPLAIN_MULTILINE_COMMENT=28, PIPE=29, QUOTED_STRING=30, 
+    INTEGER_LITERAL=31, DECIMAL_LITERAL=32, BY=33, AND=34, ASC=35, ASSIGN=36, 
+    CAST_OP=37, COMMA=38, DESC=39, DOT=40, FALSE=41, FIRST=42, LAST=43, LP=44, 
+    IN=45, IS=46, LIKE=47, NOT=48, NULL=49, NULLS=50, OR=51, PARAM=52, RLIKE=53, 
+    RP=54, TRUE=55, EQ=56, CIEQ=57, NEQ=58, LT=59, LTE=60, GT=61, GTE=62, 
+    PLUS=63, MINUS=64, ASTERISK=65, SLASH=66, PERCENT=67, OPENING_BRACKET=68, 
+    CLOSING_BRACKET=69, UNQUOTED_IDENTIFIER=70, QUOTED_IDENTIFIER=71, EXPR_LINE_COMMENT=72, 
+    EXPR_MULTILINE_COMMENT=73, EXPR_WS=74, METADATA=75, FROM_LINE_COMMENT=76, 
+    FROM_MULTILINE_COMMENT=77, FROM_WS=78, ID_PATTERN=79, PROJECT_LINE_COMMENT=80, 
+    PROJECT_MULTILINE_COMMENT=81, PROJECT_WS=82, AS=83, RENAME_LINE_COMMENT=84, 
+    RENAME_MULTILINE_COMMENT=85, RENAME_WS=86, ON=87, WITH=88, ENRICH_POLICY_NAME=89, 
+    ENRICH_LINE_COMMENT=90, ENRICH_MULTILINE_COMMENT=91, ENRICH_WS=92, ENRICH_FIELD_LINE_COMMENT=93, 
+    ENRICH_FIELD_MULTILINE_COMMENT=94, ENRICH_FIELD_WS=95, LOOKUP_LINE_COMMENT=96, 
+    LOOKUP_MULTILINE_COMMENT=97, LOOKUP_WS=98, LOOKUP_FIELD_LINE_COMMENT=99, 
+    LOOKUP_FIELD_MULTILINE_COMMENT=100, LOOKUP_FIELD_WS=101, MVEXPAND_LINE_COMMENT=102, 
+    MVEXPAND_MULTILINE_COMMENT=103, MVEXPAND_WS=104, INFO=105, SHOW_LINE_COMMENT=106, 
+    SHOW_MULTILINE_COMMENT=107, SHOW_WS=108, FUNCTIONS=109, META_LINE_COMMENT=110, 
+    META_MULTILINE_COMMENT=111, META_WS=112, COLON=113, SETTING=114, SETTING_LINE_COMMENT=115, 
+    SETTTING_MULTILINE_COMMENT=116, SETTING_WS=117, METRICS_LINE_COMMENT=118, 
+    METRICS_MULTILINE_COMMENT=119, METRICS_WS=120, CLOSING_METRICS_LINE_COMMENT=121, 
+    CLOSING_METRICS_MULTILINE_COMMENT=122, CLOSING_METRICS_WS=123;
   public static final int
     RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, 
     RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, 
@@ -49,15 +51,16 @@ public class EsqlBaseParser extends Parser {
     RULE_metadata = 17, RULE_metadataOption = 18, RULE_deprecated_metadata = 19, 
     RULE_metricsCommand = 20, RULE_evalCommand = 21, RULE_statsCommand = 22, 
     RULE_inlinestatsCommand = 23, RULE_qualifiedName = 24, RULE_qualifiedNamePattern = 25, 
-    RULE_identifier = 26, RULE_identifierPattern = 27, RULE_constant = 28, 
-    RULE_limitCommand = 29, RULE_sortCommand = 30, RULE_orderExpression = 31, 
-    RULE_keepCommand = 32, RULE_dropCommand = 33, RULE_renameCommand = 34, 
-    RULE_renameClause = 35, RULE_dissectCommand = 36, RULE_grokCommand = 37, 
-    RULE_mvExpandCommand = 38, RULE_commandOptions = 39, RULE_commandOption = 40, 
-    RULE_booleanValue = 41, RULE_numericValue = 42, RULE_decimalValue = 43, 
-    RULE_integerValue = 44, RULE_string = 45, RULE_comparisonOperator = 46, 
-    RULE_explainCommand = 47, RULE_subqueryExpression = 48, RULE_showCommand = 49, 
-    RULE_metaCommand = 50, RULE_enrichCommand = 51, RULE_enrichWithClause = 52;
+    RULE_qualifiedNamePatterns = 26, RULE_identifier = 27, RULE_identifierPattern = 28, 
+    RULE_constant = 29, RULE_limitCommand = 30, RULE_sortCommand = 31, RULE_orderExpression = 32, 
+    RULE_keepCommand = 33, RULE_dropCommand = 34, RULE_renameCommand = 35, 
+    RULE_renameClause = 36, RULE_dissectCommand = 37, RULE_grokCommand = 38, 
+    RULE_mvExpandCommand = 39, RULE_commandOptions = 40, RULE_commandOption = 41, 
+    RULE_booleanValue = 42, RULE_numericValue = 43, RULE_decimalValue = 44, 
+    RULE_integerValue = 45, RULE_string = 46, RULE_comparisonOperator = 47, 
+    RULE_explainCommand = 48, RULE_subqueryExpression = 49, RULE_showCommand = 50, 
+    RULE_metaCommand = 51, RULE_enrichCommand = 52, RULE_enrichWithClause = 53, 
+    RULE_lookupCommand = 54;
   private static String[] makeRuleNames() {
     return new String[] {
       "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", 
@@ -65,13 +68,13 @@ private static String[] makeRuleNames() {
       "primaryExpression", "functionExpression", "dataType", "rowCommand", 
       "fields", "field", "fromCommand", "indexIdentifier", "metadata", "metadataOption", 
       "deprecated_metadata", "metricsCommand", "evalCommand", "statsCommand", 
-      "inlinestatsCommand", "qualifiedName", "qualifiedNamePattern", "identifier", 
-      "identifierPattern", "constant", "limitCommand", "sortCommand", "orderExpression", 
-      "keepCommand", "dropCommand", "renameCommand", "renameClause", "dissectCommand", 
-      "grokCommand", "mvExpandCommand", "commandOptions", "commandOption", 
-      "booleanValue", "numericValue", "decimalValue", "integerValue", "string", 
-      "comparisonOperator", "explainCommand", "subqueryExpression", "showCommand", 
-      "metaCommand", "enrichCommand", "enrichWithClause"
+      "inlinestatsCommand", "qualifiedName", "qualifiedNamePattern", "qualifiedNamePatterns", 
+      "identifier", "identifierPattern", "constant", "limitCommand", "sortCommand", 
+      "orderExpression", "keepCommand", "dropCommand", "renameCommand", "renameClause", 
+      "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", 
+      "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", 
+      "string", "comparisonOperator", "explainCommand", "subqueryExpression", 
+      "showCommand", "metaCommand", "enrichCommand", "enrichWithClause", "lookupCommand"
     };
   }
   public static final String[] ruleNames = makeRuleNames();
@@ -79,25 +82,26 @@ private static String[] makeRuleNames() {
   private static String[] makeLiteralNames() {
     return new String[] {
       null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", 
-      "'grok'", "'inlinestats'", "'keep'", "'limit'", "'meta'", "'metrics'", 
-      "'mv_expand'", "'rename'", "'row'", "'show'", "'sort'", "'stats'", "'where'", 
-      null, null, null, null, null, null, null, null, "'|'", null, null, null, 
-      "'by'", "'and'", "'asc'", "'='", "'::'", "','", "'desc'", "'.'", "'false'", 
-      "'first'", "'last'", "'('", "'in'", "'is'", "'like'", "'not'", "'null'", 
-      "'nulls'", "'or'", "'?'", "'rlike'", "')'", "'true'", "'=='", "'=~'", 
-      "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", 
-      null, "']'", null, null, null, null, null, "'metadata'", null, null, 
-      null, null, null, null, null, "'as'", null, null, null, "'on'", "'with'", 
-      null, null, null, null, null, null, null, null, null, null, "'info'", 
-      null, null, null, "'functions'", null, null, null, "':'"
+      "'grok'", "'inlinestats'", "'keep'", "'limit'", "'lookup'", "'meta'", 
+      "'metrics'", "'mv_expand'", "'rename'", "'row'", "'show'", "'sort'", 
+      "'stats'", "'where'", null, null, null, null, null, null, null, null, 
+      "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "','", 
+      "'desc'", "'.'", "'false'", "'first'", "'last'", "'('", "'in'", "'is'", 
+      "'like'", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", 
+      "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", 
+      "'-'", "'*'", "'/'", "'%'", null, "']'", null, null, null, null, null, 
+      "'metadata'", null, null, null, null, null, null, null, "'as'", null, 
+      null, null, "'on'", "'with'", null, null, null, null, null, null, null, 
+      null, null, null, null, null, null, null, null, null, "'info'", null, 
+      null, null, "'functions'", null, null, null, "':'"
     };
   }
   private static final String[] _LITERAL_NAMES = makeLiteralNames();
   private static String[] makeSymbolicNames() {
     return new String[] {
       null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", 
-      "INLINESTATS", "KEEP", "LIMIT", "META", "METRICS", "MV_EXPAND", "RENAME", 
-      "ROW", "SHOW", "SORT", "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", 
+      "INLINESTATS", "KEEP", "LIMIT", "LOOKUP", "META", "METRICS", "MV_EXPAND", 
+      "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", "UNKNOWN_CMD", "LINE_COMMENT", 
       "MULTILINE_COMMENT", "WS", "INDEX_UNQUOTED_IDENTIFIER", "EXPLAIN_WS", 
       "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", "PIPE", "QUOTED_STRING", 
       "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", 
@@ -111,7 +115,9 @@ private static String[] makeSymbolicNames() {
       "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", "RENAME_WS", 
       "ON", "WITH", "ENRICH_POLICY_NAME", "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", 
       "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", 
-      "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", 
+      "ENRICH_FIELD_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", 
+      "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", 
+      "LOOKUP_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", 
       "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", 
       "SHOW_WS", "FUNCTIONS", "META_LINE_COMMENT", "META_MULTILINE_COMMENT", 
       "META_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", "SETTTING_MULTILINE_COMMENT", 
@@ -204,9 +210,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(106);
+      setState(110);
       query(0);
-      setState(107);
+      setState(111);
       match(EOF);
       }
     }
@@ -302,11 +308,11 @@ private QueryContext query(int _p) throws RecognitionException {
       _ctx = _localctx;
       _prevctx = _localctx;
 
-      setState(110);
+      setState(114);
       sourceCommand();
       }
       _ctx.stop = _input.LT(-1);
-      setState(117);
+      setState(121);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -317,16 +323,16 @@ private QueryContext query(int _p) throws RecognitionException {
           {
           _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState));
           pushNewRecursionContext(_localctx, _startState, RULE_query);
-          setState(112);
+          setState(116);
           if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-          setState(113);
+          setState(117);
           match(PIPE);
-          setState(114);
+          setState(118);
           processingCommand();
           }
           } 
         }
-        setState(119);
+        setState(123);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       }
@@ -387,48 +393,48 @@ public final SourceCommandContext sourceCommand() throws RecognitionException {
     SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState());
     enterRule(_localctx, 4, RULE_sourceCommand);
     try {
-      setState(126);
+      setState(130);
       _errHandler.sync(this);
       switch (_input.LA(1)) {
       case EXPLAIN:
         enterOuterAlt(_localctx, 1);
         {
-        setState(120);
+        setState(124);
         explainCommand();
         }
         break;
       case FROM:
         enterOuterAlt(_localctx, 2);
         {
-        setState(121);
+        setState(125);
         fromCommand();
         }
         break;
       case ROW:
         enterOuterAlt(_localctx, 3);
         {
-        setState(122);
+        setState(126);
         rowCommand();
         }
         break;
       case METRICS:
         enterOuterAlt(_localctx, 4);
         {
-        setState(123);
+        setState(127);
         metricsCommand();
         }
         break;
       case SHOW:
         enterOuterAlt(_localctx, 5);
         {
-        setState(124);
+        setState(128);
         showCommand();
         }
         break;
       case META:
         enterOuterAlt(_localctx, 6);
         {
-        setState(125);
+        setState(129);
         metaCommand();
         }
         break;
@@ -458,6 +464,9 @@ public InlinestatsCommandContext inlinestatsCommand() {
     public LimitCommandContext limitCommand() {
       return getRuleContext(LimitCommandContext.class,0);
     }
+    public LookupCommandContext lookupCommand() {
+      return getRuleContext(LookupCommandContext.class,0);
+    }
     public KeepCommandContext keepCommand() {
       return getRuleContext(KeepCommandContext.class,0);
     }
@@ -512,97 +521,104 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce
     ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState());
     enterRule(_localctx, 6, RULE_processingCommand);
     try {
-      setState(141);
+      setState(146);
       _errHandler.sync(this);
       switch (_input.LA(1)) {
       case EVAL:
         enterOuterAlt(_localctx, 1);
         {
-        setState(128);
+        setState(132);
         evalCommand();
         }
         break;
       case INLINESTATS:
         enterOuterAlt(_localctx, 2);
         {
-        setState(129);
+        setState(133);
         inlinestatsCommand();
         }
         break;
       case LIMIT:
         enterOuterAlt(_localctx, 3);
         {
-        setState(130);
+        setState(134);
         limitCommand();
         }
         break;
-      case KEEP:
+      case LOOKUP:
         enterOuterAlt(_localctx, 4);
         {
-        setState(131);
+        setState(135);
+        lookupCommand();
+        }
+        break;
+      case KEEP:
+        enterOuterAlt(_localctx, 5);
+        {
+        setState(136);
         keepCommand();
         }
         break;
       case SORT:
-        enterOuterAlt(_localctx, 5);
+        enterOuterAlt(_localctx, 6);
         {
-        setState(132);
+        setState(137);
         sortCommand();
         }
         break;
       case STATS:
-        enterOuterAlt(_localctx, 6);
+        enterOuterAlt(_localctx, 7);
         {
-        setState(133);
+        setState(138);
         statsCommand();
         }
         break;
       case WHERE:
-        enterOuterAlt(_localctx, 7);
+        enterOuterAlt(_localctx, 8);
         {
-        setState(134);
+        setState(139);
         whereCommand();
         }
         break;
       case DROP:
-        enterOuterAlt(_localctx, 8);
+        enterOuterAlt(_localctx, 9);
         {
-        setState(135);
+        setState(140);
         dropCommand();
         }
         break;
       case RENAME:
-        enterOuterAlt(_localctx, 9);
+        enterOuterAlt(_localctx, 10);
         {
-        setState(136);
+        setState(141);
         renameCommand();
         }
         break;
       case DISSECT:
-        enterOuterAlt(_localctx, 10);
+        enterOuterAlt(_localctx, 11);
         {
-        setState(137);
+        setState(142);
         dissectCommand();
         }
         break;
       case GROK:
-        enterOuterAlt(_localctx, 11);
+        enterOuterAlt(_localctx, 12);
         {
-        setState(138);
+        setState(143);
         grokCommand();
         }
         break;
       case ENRICH:
-        enterOuterAlt(_localctx, 12);
+        enterOuterAlt(_localctx, 13);
         {
-        setState(139);
+        setState(144);
         enrichCommand();
         }
         break;
       case MV_EXPAND:
-        enterOuterAlt(_localctx, 13);
+        enterOuterAlt(_localctx, 14);
         {
-        setState(140);
+        setState(145);
         mvExpandCommand();
         }
         break;
@@ -653,9 +669,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(143);
+      setState(148);
       match(WHERE);
-      setState(144);
+      setState(149);
       booleanExpression(0);
       }
     }
@@ -850,7 +866,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(174);
+      setState(179);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
       case 1:
@@ -859,9 +875,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(147);
+        setState(152);
         match(NOT);
-        setState(148);
+        setState(153);
         booleanExpression(7);
         }
         break;
@@ -870,7 +886,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new BooleanDefaultContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(149);
+        setState(154);
         valueExpression();
         }
         break;
@@ -879,7 +895,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new RegexExpressionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(150);
+        setState(155);
         regexBooleanExpression();
         }
         break;
@@ -888,41 +904,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new LogicalInContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(151);
+        setState(156);
         valueExpression();
-        setState(153);
+        setState(158);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(152);
+          setState(157);
           match(NOT);
           }
         }
 
-        setState(155);
+        setState(160);
         match(IN);
-        setState(156);
+        setState(161);
         match(LP);
-        setState(157);
-        valueExpression();
         setState(162);
+        valueExpression();
+        setState(167);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(158);
+          setState(163);
           match(COMMA);
-          setState(159);
+          setState(164);
           valueExpression();
           }
           }
-          setState(164);
+          setState(169);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(165);
+        setState(170);
         match(RP);
         }
         break;
@@ -931,27 +947,27 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         _localctx = new IsNullContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(167);
+        setState(172);
         valueExpression();
-        setState(168);
+        setState(173);
         match(IS);
-        setState(170);
+        setState(175);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(169);
+          setState(174);
           match(NOT);
           }
         }
 
-        setState(172);
+        setState(177);
         match(NULL);
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(184);
+      setState(189);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,8,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -959,7 +975,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(182);
+          setState(187);
           _errHandler.sync(this);
           switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
           case 1:
@@ -967,11 +983,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
             _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState));
             ((LogicalBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression);
-            setState(176);
+            setState(181);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(177);
+            setState(182);
             ((LogicalBinaryContext)_localctx).operator = match(AND);
-            setState(178);
+            setState(183);
             ((LogicalBinaryContext)_localctx).right = booleanExpression(5);
             }
             break;
@@ -980,18 +996,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
             _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState));
             ((LogicalBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression);
-            setState(179);
+            setState(184);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(180);
+            setState(185);
             ((LogicalBinaryContext)_localctx).operator = match(OR);
-            setState(181);
+            setState(186);
             ((LogicalBinaryContext)_localctx).right = booleanExpression(4);
             }
             break;
           }
           } 
         }
-        setState(186);
+        setState(191);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,8,_ctx);
       }
@@ -1046,48 +1062,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog
     enterRule(_localctx, 12, RULE_regexBooleanExpression);
     int _la;
     try {
-      setState(201);
+      setState(206);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(187);
+        setState(192);
         valueExpression();
-        setState(189);
+        setState(194);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(188);
+          setState(193);
           match(NOT);
           }
         }
 
-        setState(191);
+        setState(196);
         ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE);
-        setState(192);
+        setState(197);
         ((RegexBooleanExpressionContext)_localctx).pattern = string();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(194);
+        setState(199);
         valueExpression();
-        setState(196);
+        setState(201);
         _errHandler.sync(this);
         _la = _input.LA(1);
         if (_la==NOT) {
           {
-          setState(195);
+          setState(200);
           match(NOT);
           }
         }
 
-        setState(198);
+        setState(203);
         ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE);
-        setState(199);
+        setState(204);
         ((RegexBooleanExpressionContext)_localctx).pattern = string();
         }
         break;
@@ -1173,14 +1189,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio
     ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState());
     enterRule(_localctx, 14, RULE_valueExpression);
     try {
-      setState(208);
+      setState(213);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
       case 1:
         _localctx = new ValueExpressionDefaultContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(203);
+        setState(208);
         operatorExpression(0);
         }
         break;
@@ -1188,11 +1204,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio
         _localctx = new ComparisonContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(204);
+        setState(209);
         ((ComparisonContext)_localctx).left = operatorExpression(0);
-        setState(205);
+        setState(210);
         comparisonOperator();
-        setState(206);
+        setState(211);
         ((ComparisonContext)_localctx).right = operatorExpression(0);
         }
         break;
@@ -1317,7 +1333,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(214);
+      setState(219);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) {
       case 1:
@@ -1326,7 +1342,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(211);
+        setState(216);
         primaryExpression(0);
         }
         break;
@@ -1335,7 +1351,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
         _localctx = new ArithmeticUnaryContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(212);
+        setState(217);
         ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
@@ -1346,13 +1362,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
           _errHandler.reportMatch(this);
           consume();
         }
-        setState(213);
+        setState(218);
         operatorExpression(3);
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(224);
+      setState(229);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,15,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1360,7 +1376,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(222);
+          setState(227);
           _errHandler.sync(this);
           switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) {
           case 1:
@@ -1368,12 +1384,12 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
             _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState));
             ((ArithmeticBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression);
-            setState(216);
+            setState(221);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(217);
+            setState(222);
             ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1);
             _la = _input.LA(1);
-            if ( !(((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & 7L) != 0)) ) {
+            if ( !(((((_la - 65)) & ~0x3f) == 0 && ((1L << (_la - 65)) & 7L) != 0)) ) {
               ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this);
             }
             else {
@@ -1381,7 +1397,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
               _errHandler.reportMatch(this);
               consume();
             }
-            setState(218);
+            setState(223);
             ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3);
             }
             break;
@@ -1390,9 +1406,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
             _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState));
             ((ArithmeticBinaryContext)_localctx).left = _prevctx;
             pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression);
-            setState(219);
+            setState(224);
             if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-            setState(220);
+            setState(225);
             ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1);
             _la = _input.LA(1);
             if ( !(_la==PLUS || _la==MINUS) ) {
@@ -1403,14 +1419,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
               _errHandler.reportMatch(this);
               consume();
             }
-            setState(221);
+            setState(226);
             ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2);
             }
             break;
           }
           } 
         }
-        setState(226);
+        setState(231);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,15,_ctx);
       }
@@ -1568,7 +1584,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(235);
+      setState(240);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
       case 1:
@@ -1577,7 +1593,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(228);
+        setState(233);
         constant();
         }
         break;
@@ -1586,7 +1602,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new DereferenceContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(229);
+        setState(234);
         qualifiedName();
         }
         break;
@@ -1595,7 +1611,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new FunctionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(230);
+        setState(235);
         functionExpression();
         }
         break;
@@ -1604,17 +1620,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         _localctx = new ParenthesizedExpressionContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(231);
+        setState(236);
         match(LP);
-        setState(232);
+        setState(237);
         booleanExpression(0);
-        setState(233);
+        setState(238);
         match(RP);
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(242);
+      setState(247);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1625,16 +1641,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
           {
           _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState));
           pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression);
-          setState(237);
+          setState(242);
           if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)");
-          setState(238);
+          setState(243);
           match(CAST_OP);
-          setState(239);
+          setState(244);
           dataType();
           }
           } 
         }
-        setState(244);
+        setState(249);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       }
@@ -1696,16 +1712,16 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(245);
+      setState(250);
       identifier();
-      setState(246);
+      setState(251);
       match(LP);
-      setState(256);
+      setState(261);
       _errHandler.sync(this);
       switch (_input.LA(1)) {
       case ASTERISK:
         {
-        setState(247);
+        setState(252);
         match(ASTERISK);
         }
         break;
@@ -1725,21 +1741,21 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
       case QUOTED_IDENTIFIER:
         {
         {
-        setState(248);
-        booleanExpression(0);
         setState(253);
+        booleanExpression(0);
+        setState(258);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(249);
+          setState(254);
           match(COMMA);
-          setState(250);
+          setState(255);
           booleanExpression(0);
           }
           }
-          setState(255);
+          setState(260);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
@@ -1751,7 +1767,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
       default:
         break;
       }
-      setState(258);
+      setState(263);
       match(RP);
       }
     }
@@ -1809,7 +1825,7 @@ public final DataTypeContext dataType() throws RecognitionException {
       _localctx = new ToDataTypeContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(260);
+      setState(265);
       identifier();
       }
     }
@@ -1856,9 +1872,9 @@ public final RowCommandContext rowCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(262);
+      setState(267);
       match(ROW);
-      setState(263);
+      setState(268);
       fields();
       }
     }
@@ -1912,23 +1928,23 @@ public final FieldsContext fields() throws RecognitionException {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(265);
-      field();
       setState(270);
+      field();
+      setState(275);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(266);
+          setState(271);
           match(COMMA);
-          setState(267);
+          setState(272);
           field();
           }
           } 
         }
-        setState(272);
+        setState(277);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       }
@@ -1978,24 +1994,24 @@ public final FieldContext field() throws RecognitionException {
     FieldContext _localctx = new FieldContext(_ctx, getState());
     enterRule(_localctx, 28, RULE_field);
     try {
-      setState(278);
+      setState(283);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(273);
+        setState(278);
         booleanExpression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(274);
+        setState(279);
         qualifiedName();
-        setState(275);
+        setState(280);
         match(ASSIGN);
-        setState(276);
+        setState(281);
         booleanExpression(0);
         }
         break;
@@ -2055,34 +2071,34 @@ public final FromCommandContext fromCommand() throws RecognitionException {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(280);
+      setState(285);
       match(FROM);
-      setState(281);
-      indexIdentifier();
       setState(286);
+      indexIdentifier();
+      setState(291);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(282);
+          setState(287);
           match(COMMA);
-          setState(283);
+          setState(288);
           indexIdentifier();
           }
           } 
         }
-        setState(288);
+        setState(293);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
       }
-      setState(290);
+      setState(295);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
       case 1:
         {
-        setState(289);
+        setState(294);
         metadata();
         }
         break;
@@ -2129,7 +2145,7 @@ public final IndexIdentifierContext indexIdentifier() throws RecognitionExceptio
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(292);
+      setState(297);
       match(INDEX_UNQUOTED_IDENTIFIER);
       }
     }
@@ -2176,20 +2192,20 @@ public final MetadataContext metadata() throws RecognitionException {
     MetadataContext _localctx = new MetadataContext(_ctx, getState());
     enterRule(_localctx, 34, RULE_metadata);
     try {
-      setState(296);
+      setState(301);
       _errHandler.sync(this);
       switch (_input.LA(1)) {
       case METADATA:
         enterOuterAlt(_localctx, 1);
         {
-        setState(294);
+        setState(299);
         metadataOption();
         }
         break;
       case OPENING_BRACKET:
         enterOuterAlt(_localctx, 2);
         {
-        setState(295);
+        setState(300);
         deprecated_metadata();
         }
         break;
@@ -2248,25 +2264,25 @@ public final MetadataOptionContext metadataOption() throws RecognitionException
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(298);
+      setState(303);
       match(METADATA);
-      setState(299);
-      indexIdentifier();
       setState(304);
+      indexIdentifier();
+      setState(309);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(300);
+          setState(305);
           match(COMMA);
-          setState(301);
+          setState(306);
           indexIdentifier();
           }
           } 
         }
-        setState(306);
+        setState(311);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
       }
@@ -2315,11 +2331,11 @@ public final Deprecated_metadataContext deprecated_metadata() throws Recognition
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(307);
+      setState(312);
       match(OPENING_BRACKET);
-      setState(308);
+      setState(313);
       metadataOption();
-      setState(309);
+      setState(314);
       match(CLOSING_BRACKET);
       }
     }
@@ -2383,46 +2399,46 @@ public final MetricsCommandContext metricsCommand() throws RecognitionException
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(311);
+      setState(316);
       match(METRICS);
-      setState(312);
-      indexIdentifier();
       setState(317);
+      indexIdentifier();
+      setState(322);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(313);
+          setState(318);
           match(COMMA);
-          setState(314);
+          setState(319);
           indexIdentifier();
           }
           } 
         }
-        setState(319);
+        setState(324);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
       }
-      setState(321);
+      setState(326);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
       case 1:
         {
-        setState(320);
+        setState(325);
         ((MetricsCommandContext)_localctx).aggregates = fields();
         }
         break;
       }
-      setState(325);
+      setState(330);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
       case 1:
         {
-        setState(323);
+        setState(328);
         match(BY);
-        setState(324);
+        setState(329);
         ((MetricsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -2472,9 +2488,9 @@ public final EvalCommandContext evalCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(327);
+      setState(332);
       match(EVAL);
-      setState(328);
+      setState(333);
       fields();
       }
     }
@@ -2527,26 +2543,26 @@ public final StatsCommandContext statsCommand() throws RecognitionException {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(330);
+      setState(335);
       match(STATS);
-      setState(332);
+      setState(337);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         {
-        setState(331);
+        setState(336);
         ((StatsCommandContext)_localctx).stats = fields();
         }
         break;
       }
-      setState(336);
+      setState(341);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         {
-        setState(334);
+        setState(339);
         match(BY);
-        setState(335);
+        setState(340);
         ((StatsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -2602,18 +2618,18 @@ public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionEx
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(338);
+      setState(343);
       match(INLINESTATS);
-      setState(339);
+      setState(344);
       ((InlinestatsCommandContext)_localctx).stats = fields();
-      setState(342);
+      setState(347);
       _errHandler.sync(this);
       switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         {
-        setState(340);
+        setState(345);
         match(BY);
-        setState(341);
+        setState(346);
         ((InlinestatsCommandContext)_localctx).grouping = fields();
         }
         break;
@@ -2670,23 +2686,23 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(344);
-      identifier();
       setState(349);
+      identifier();
+      setState(354);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,32,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(345);
+          setState(350);
           match(DOT);
-          setState(346);
+          setState(351);
           identifier();
           }
           } 
         }
-        setState(351);
+        setState(356);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,32,_ctx);
       }
@@ -2742,23 +2758,23 @@ public final QualifiedNamePatternContext qualifiedNamePattern() throws Recogniti
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(352);
-      identifierPattern();
       setState(357);
+      identifierPattern();
+      setState(362);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(353);
+          setState(358);
           match(DOT);
-          setState(354);
+          setState(359);
           identifierPattern();
           }
           } 
         }
-        setState(359);
+        setState(364);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,33,_ctx);
       }
@@ -2775,6 +2791,78 @@ public final QualifiedNamePatternContext qualifiedNamePattern() throws Recogniti
     return _localctx;
   }
 
+  @SuppressWarnings("CheckReturnValue")
+  public static class QualifiedNamePatternsContext extends ParserRuleContext {
+    public List<QualifiedNamePatternContext> qualifiedNamePattern() {
+      return getRuleContexts(QualifiedNamePatternContext.class);
+    }
+    public QualifiedNamePatternContext qualifiedNamePattern(int i) {
+      return getRuleContext(QualifiedNamePatternContext.class,i);
+    }
+    public List<TerminalNode> COMMA() { return getTokens(EsqlBaseParser.COMMA); }
+    public TerminalNode COMMA(int i) {
+      return getToken(EsqlBaseParser.COMMA, i);
+    }
+    @SuppressWarnings("this-escape")
+    public QualifiedNamePatternsContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_qualifiedNamePatterns; }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterQualifiedNamePatterns(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitQualifiedNamePatterns(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitQualifiedNamePatterns(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final QualifiedNamePatternsContext qualifiedNamePatterns() throws RecognitionException {
+    QualifiedNamePatternsContext _localctx = new QualifiedNamePatternsContext(_ctx, getState());
+    enterRule(_localctx, 52, RULE_qualifiedNamePatterns);
+    try {
+      int _alt;
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(365);
+      qualifiedNamePattern();
+      setState(370);
+      _errHandler.sync(this);
+      _alt = getInterpreter().adaptivePredict(_input,34,_ctx);
+      while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+        if ( _alt==1 ) {
+          {
+          {
+          setState(366);
+          match(COMMA);
+          setState(367);
+          qualifiedNamePattern();
+          }
+          } 
+        }
+        setState(372);
+        _errHandler.sync(this);
+        _alt = getInterpreter().adaptivePredict(_input,34,_ctx);
+      }
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   @SuppressWarnings("CheckReturnValue")
   public static class IdentifierContext extends ParserRuleContext {
     public TerminalNode UNQUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.UNQUOTED_IDENTIFIER, 0); }
@@ -2801,12 +2889,12 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IdentifierContext identifier() throws RecognitionException {
     IdentifierContext _localctx = new IdentifierContext(_ctx, getState());
-    enterRule(_localctx, 52, RULE_identifier);
+    enterRule(_localctx, 54, RULE_identifier);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(360);
+      setState(373);
       _la = _input.LA(1);
       if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) {
       _errHandler.recoverInline(this);
@@ -2854,11 +2942,11 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IdentifierPatternContext identifierPattern() throws RecognitionException {
     IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState());
-    enterRule(_localctx, 54, RULE_identifierPattern);
+    enterRule(_localctx, 56, RULE_identifierPattern);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(362);
+      setState(375);
       match(ID_PATTERN);
       }
     }
@@ -3124,17 +3212,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ConstantContext constant() throws RecognitionException {
     ConstantContext _localctx = new ConstantContext(_ctx, getState());
-    enterRule(_localctx, 56, RULE_constant);
+    enterRule(_localctx, 58, RULE_constant);
     int _la;
     try {
-      setState(406);
+      setState(419);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) {
       case 1:
         _localctx = new NullLiteralContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(364);
+        setState(377);
         match(NULL);
         }
         break;
@@ -3142,9 +3230,9 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new QualifiedIntegerLiteralContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(365);
+        setState(378);
         integerValue();
-        setState(366);
+        setState(379);
         match(UNQUOTED_IDENTIFIER);
         }
         break;
@@ -3152,7 +3240,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new DecimalLiteralContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(368);
+        setState(381);
         decimalValue();
         }
         break;
@@ -3160,7 +3248,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new IntegerLiteralContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(369);
+        setState(382);
         integerValue();
         }
         break;
@@ -3168,7 +3256,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new BooleanLiteralContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(370);
+        setState(383);
         booleanValue();
         }
         break;
@@ -3176,7 +3264,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new InputParamContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(371);
+        setState(384);
         match(PARAM);
         }
         break;
@@ -3184,7 +3272,7 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new StringLiteralContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(372);
+        setState(385);
         string();
         }
         break;
@@ -3192,27 +3280,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new NumericArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(373);
+        setState(386);
         match(OPENING_BRACKET);
-        setState(374);
+        setState(387);
         numericValue();
-        setState(379);
+        setState(392);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(375);
+          setState(388);
           match(COMMA);
-          setState(376);
+          setState(389);
           numericValue();
           }
           }
-          setState(381);
+          setState(394);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(382);
+        setState(395);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3220,27 +3308,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new BooleanArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(384);
+        setState(397);
         match(OPENING_BRACKET);
-        setState(385);
+        setState(398);
         booleanValue();
-        setState(390);
+        setState(403);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(386);
+          setState(399);
           match(COMMA);
-          setState(387);
+          setState(400);
           booleanValue();
           }
           }
-          setState(392);
+          setState(405);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(393);
+        setState(406);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3248,27 +3336,27 @@ public final ConstantContext constant() throws RecognitionException {
         _localctx = new StringArrayLiteralContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(395);
+        setState(408);
         match(OPENING_BRACKET);
-        setState(396);
+        setState(409);
         string();
-        setState(401);
+        setState(414);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(397);
+          setState(410);
           match(COMMA);
-          setState(398);
+          setState(411);
           string();
           }
           }
-          setState(403);
+          setState(416);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(404);
+        setState(417);
         match(CLOSING_BRACKET);
         }
         break;
@@ -3311,13 +3399,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final LimitCommandContext limitCommand() throws RecognitionException {
     LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState());
-    enterRule(_localctx, 58, RULE_limitCommand);
+    enterRule(_localctx, 60, RULE_limitCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(408);
+      setState(421);
       match(LIMIT);
-      setState(409);
+      setState(422);
       match(INTEGER_LITERAL);
       }
     }
@@ -3367,32 +3455,32 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final SortCommandContext sortCommand() throws RecognitionException {
     SortCommandContext _localctx = new SortCommandContext(_ctx, getState());
-    enterRule(_localctx, 60, RULE_sortCommand);
+    enterRule(_localctx, 62, RULE_sortCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(411);
+      setState(424);
       match(SORT);
-      setState(412);
+      setState(425);
       orderExpression();
-      setState(417);
+      setState(430);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,38,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,39,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(413);
+          setState(426);
           match(COMMA);
-          setState(414);
+          setState(427);
           orderExpression();
           }
           } 
         }
-        setState(419);
+        setState(432);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,38,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,39,_ctx);
       }
       }
     }
@@ -3441,19 +3529,19 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final OrderExpressionContext orderExpression() throws RecognitionException {
     OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState());
-    enterRule(_localctx, 62, RULE_orderExpression);
+    enterRule(_localctx, 64, RULE_orderExpression);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(420);
+      setState(433);
       booleanExpression(0);
-      setState(422);
+      setState(435);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) {
       case 1:
         {
-        setState(421);
+        setState(434);
         ((OrderExpressionContext)_localctx).ordering = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==ASC || _la==DESC) ) {
@@ -3467,14 +3555,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio
         }
         break;
       }
-      setState(426);
+      setState(439);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) {
       case 1:
         {
-        setState(424);
+        setState(437);
         match(NULLS);
-        setState(425);
+        setState(438);
         ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1);
         _la = _input.LA(1);
         if ( !(_la==FIRST || _la==LAST) ) {
@@ -3504,15 +3592,8 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio
   @SuppressWarnings("CheckReturnValue")
   public static class KeepCommandContext extends ParserRuleContext {
     public TerminalNode KEEP() { return getToken(EsqlBaseParser.KEEP, 0); }
-    public List<QualifiedNamePatternContext> qualifiedNamePattern() {
-      return getRuleContexts(QualifiedNamePatternContext.class);
-    }
-    public QualifiedNamePatternContext qualifiedNamePattern(int i) {
-      return getRuleContext(QualifiedNamePatternContext.class,i);
-    }
-    public List<TerminalNode> COMMA() { return getTokens(EsqlBaseParser.COMMA); }
-    public TerminalNode COMMA(int i) {
-      return getToken(EsqlBaseParser.COMMA, i);
+    public QualifiedNamePatternsContext qualifiedNamePatterns() {
+      return getRuleContext(QualifiedNamePatternsContext.class,0);
     }
     @SuppressWarnings("this-escape")
     public KeepCommandContext(ParserRuleContext parent, int invokingState) {
@@ -3536,33 +3617,14 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final KeepCommandContext keepCommand() throws RecognitionException {
     KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState());
-    enterRule(_localctx, 64, RULE_keepCommand);
+    enterRule(_localctx, 66, RULE_keepCommand);
     try {
-      int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(428);
+      setState(441);
       match(KEEP);
-      setState(429);
-      qualifiedNamePattern();
-      setState(434);
-      _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,41,_ctx);
-      while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
-        if ( _alt==1 ) {
-          {
-          {
-          setState(430);
-          match(COMMA);
-          setState(431);
-          qualifiedNamePattern();
-          }
-          } 
-        }
-        setState(436);
-        _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,41,_ctx);
-      }
+      setState(442);
+      qualifiedNamePatterns();
       }
     }
     catch (RecognitionException re) {
@@ -3579,15 +3641,8 @@ public final KeepCommandContext keepCommand() throws RecognitionException {
   @SuppressWarnings("CheckReturnValue")
   public static class DropCommandContext extends ParserRuleContext {
     public TerminalNode DROP() { return getToken(EsqlBaseParser.DROP, 0); }
-    public List<QualifiedNamePatternContext> qualifiedNamePattern() {
-      return getRuleContexts(QualifiedNamePatternContext.class);
-    }
-    public QualifiedNamePatternContext qualifiedNamePattern(int i) {
-      return getRuleContext(QualifiedNamePatternContext.class,i);
-    }
-    public List<TerminalNode> COMMA() { return getTokens(EsqlBaseParser.COMMA); }
-    public TerminalNode COMMA(int i) {
-      return getToken(EsqlBaseParser.COMMA, i);
+    public QualifiedNamePatternsContext qualifiedNamePatterns() {
+      return getRuleContext(QualifiedNamePatternsContext.class,0);
     }
     @SuppressWarnings("this-escape")
     public DropCommandContext(ParserRuleContext parent, int invokingState) {
@@ -3611,33 +3666,14 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DropCommandContext dropCommand() throws RecognitionException {
     DropCommandContext _localctx = new DropCommandContext(_ctx, getState());
-    enterRule(_localctx, 66, RULE_dropCommand);
+    enterRule(_localctx, 68, RULE_dropCommand);
     try {
-      int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(437);
+      setState(444);
       match(DROP);
-      setState(438);
-      qualifiedNamePattern();
-      setState(443);
-      _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,42,_ctx);
-      while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
-        if ( _alt==1 ) {
-          {
-          {
-          setState(439);
-          match(COMMA);
-          setState(440);
-          qualifiedNamePattern();
-          }
-          } 
-        }
-        setState(445);
-        _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,42,_ctx);
-      }
+      setState(445);
+      qualifiedNamePatterns();
       }
     }
     catch (RecognitionException re) {
@@ -3686,32 +3722,32 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final RenameCommandContext renameCommand() throws RecognitionException {
     RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState());
-    enterRule(_localctx, 68, RULE_renameCommand);
+    enterRule(_localctx, 70, RULE_renameCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(446);
-      match(RENAME);
       setState(447);
+      match(RENAME);
+      setState(448);
       renameClause();
-      setState(452);
+      setState(453);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,43,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,42,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(448);
-          match(COMMA);
           setState(449);
+          match(COMMA);
+          setState(450);
           renameClause();
           }
           } 
         }
-        setState(454);
+        setState(455);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,43,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,42,_ctx);
       }
       }
     }
@@ -3759,15 +3795,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final RenameClauseContext renameClause() throws RecognitionException {
     RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState());
-    enterRule(_localctx, 70, RULE_renameClause);
+    enterRule(_localctx, 72, RULE_renameClause);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(455);
-      ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern();
       setState(456);
-      match(AS);
+      ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern();
       setState(457);
+      match(AS);
+      setState(458);
       ((RenameClauseContext)_localctx).newName = qualifiedNamePattern();
       }
     }
@@ -3816,22 +3852,22 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DissectCommandContext dissectCommand() throws RecognitionException {
     DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState());
-    enterRule(_localctx, 72, RULE_dissectCommand);
+    enterRule(_localctx, 74, RULE_dissectCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(459);
-      match(DISSECT);
       setState(460);
-      primaryExpression(0);
+      match(DISSECT);
       setState(461);
+      primaryExpression(0);
+      setState(462);
       string();
-      setState(463);
+      setState(464);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) {
       case 1:
         {
-        setState(462);
+        setState(463);
         commandOptions();
         }
         break;
@@ -3880,15 +3916,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final GrokCommandContext grokCommand() throws RecognitionException {
     GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState());
-    enterRule(_localctx, 74, RULE_grokCommand);
+    enterRule(_localctx, 76, RULE_grokCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(465);
-      match(GROK);
       setState(466);
-      primaryExpression(0);
+      match(GROK);
       setState(467);
+      primaryExpression(0);
+      setState(468);
       string();
       }
     }
@@ -3931,13 +3967,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MvExpandCommandContext mvExpandCommand() throws RecognitionException {
     MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState());
-    enterRule(_localctx, 76, RULE_mvExpandCommand);
+    enterRule(_localctx, 78, RULE_mvExpandCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(469);
-      match(MV_EXPAND);
       setState(470);
+      match(MV_EXPAND);
+      setState(471);
       qualifiedName();
       }
     }
@@ -3986,30 +4022,30 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final CommandOptionsContext commandOptions() throws RecognitionException {
     CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState());
-    enterRule(_localctx, 78, RULE_commandOptions);
+    enterRule(_localctx, 80, RULE_commandOptions);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(472);
+      setState(473);
       commandOption();
-      setState(477);
+      setState(478);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,45,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,44,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(473);
-          match(COMMA);
           setState(474);
+          match(COMMA);
+          setState(475);
           commandOption();
           }
           } 
         }
-        setState(479);
+        setState(480);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,45,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,44,_ctx);
       }
       }
     }
@@ -4055,15 +4091,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final CommandOptionContext commandOption() throws RecognitionException {
     CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState());
-    enterRule(_localctx, 80, RULE_commandOption);
+    enterRule(_localctx, 82, RULE_commandOption);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(480);
-      identifier();
       setState(481);
-      match(ASSIGN);
+      identifier();
       setState(482);
+      match(ASSIGN);
+      setState(483);
       constant();
       }
     }
@@ -4104,12 +4140,12 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final BooleanValueContext booleanValue() throws RecognitionException {
     BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState());
-    enterRule(_localctx, 82, RULE_booleanValue);
+    enterRule(_localctx, 84, RULE_booleanValue);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(484);
+      setState(485);
       _la = _input.LA(1);
       if ( !(_la==FALSE || _la==TRUE) ) {
       _errHandler.recoverInline(this);
@@ -4162,22 +4198,22 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final NumericValueContext numericValue() throws RecognitionException {
     NumericValueContext _localctx = new NumericValueContext(_ctx, getState());
-    enterRule(_localctx, 84, RULE_numericValue);
+    enterRule(_localctx, 86, RULE_numericValue);
     try {
-      setState(488);
+      setState(489);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(486);
+        setState(487);
         decimalValue();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(487);
+        setState(488);
         integerValue();
         }
         break;
@@ -4221,17 +4257,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final DecimalValueContext decimalValue() throws RecognitionException {
     DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState());
-    enterRule(_localctx, 86, RULE_decimalValue);
+    enterRule(_localctx, 88, RULE_decimalValue);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(491);
+      setState(492);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if (_la==PLUS || _la==MINUS) {
         {
-        setState(490);
+        setState(491);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
         _errHandler.recoverInline(this);
@@ -4244,7 +4280,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException {
         }
       }
 
-      setState(493);
+      setState(494);
       match(DECIMAL_LITERAL);
       }
     }
@@ -4286,17 +4322,17 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final IntegerValueContext integerValue() throws RecognitionException {
     IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState());
-    enterRule(_localctx, 88, RULE_integerValue);
+    enterRule(_localctx, 90, RULE_integerValue);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(496);
+      setState(497);
       _errHandler.sync(this);
       _la = _input.LA(1);
       if (_la==PLUS || _la==MINUS) {
         {
-        setState(495);
+        setState(496);
         _la = _input.LA(1);
         if ( !(_la==PLUS || _la==MINUS) ) {
         _errHandler.recoverInline(this);
@@ -4309,7 +4345,7 @@ public final IntegerValueContext integerValue() throws RecognitionException {
         }
       }
 
-      setState(498);
+      setState(499);
       match(INTEGER_LITERAL);
       }
     }
@@ -4349,11 +4385,11 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final StringContext string() throws RecognitionException {
     StringContext _localctx = new StringContext(_ctx, getState());
-    enterRule(_localctx, 90, RULE_string);
+    enterRule(_localctx, 92, RULE_string);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(500);
+      setState(501);
       match(QUOTED_STRING);
       }
     }
@@ -4398,14 +4434,14 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ComparisonOperatorContext comparisonOperator() throws RecognitionException {
     ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState());
-    enterRule(_localctx, 92, RULE_comparisonOperator);
+    enterRule(_localctx, 94, RULE_comparisonOperator);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(502);
+      setState(503);
       _la = _input.LA(1);
-      if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 4503599627370496000L) != 0)) ) {
+      if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 9007199254740992000L) != 0)) ) {
       _errHandler.recoverInline(this);
       }
       else {
@@ -4454,13 +4490,13 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ExplainCommandContext explainCommand() throws RecognitionException {
     ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState());
-    enterRule(_localctx, 94, RULE_explainCommand);
+    enterRule(_localctx, 96, RULE_explainCommand);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(504);
-      match(EXPLAIN);
       setState(505);
+      match(EXPLAIN);
+      setState(506);
       subqueryExpression();
       }
     }
@@ -4504,15 +4540,15 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final SubqueryExpressionContext subqueryExpression() throws RecognitionException {
     SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState());
-    enterRule(_localctx, 96, RULE_subqueryExpression);
+    enterRule(_localctx, 98, RULE_subqueryExpression);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(507);
-      match(OPENING_BRACKET);
       setState(508);
-      query(0);
+      match(OPENING_BRACKET);
       setState(509);
+      query(0);
+      setState(510);
       match(CLOSING_BRACKET);
       }
     }
@@ -4564,14 +4600,14 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final ShowCommandContext showCommand() throws RecognitionException {
     ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState());
-    enterRule(_localctx, 98, RULE_showCommand);
+    enterRule(_localctx, 100, RULE_showCommand);
     try {
       _localctx = new ShowInfoContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(511);
-      match(SHOW);
       setState(512);
+      match(SHOW);
+      setState(513);
       match(INFO);
       }
     }
@@ -4623,14 +4659,14 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final MetaCommandContext metaCommand() throws RecognitionException {
     MetaCommandContext _localctx = new MetaCommandContext(_ctx, getState());
-    enterRule(_localctx, 100, RULE_metaCommand);
+    enterRule(_localctx, 102, RULE_metaCommand);
     try {
       _localctx = new MetaFunctionsContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(514);
-      match(META);
       setState(515);
+      match(META);
+      setState(516);
       match(FUNCTIONS);
       }
     }
@@ -4688,53 +4724,53 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final EnrichCommandContext enrichCommand() throws RecognitionException {
     EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState());
-    enterRule(_localctx, 102, RULE_enrichCommand);
+    enterRule(_localctx, 104, RULE_enrichCommand);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(517);
-      match(ENRICH);
       setState(518);
+      match(ENRICH);
+      setState(519);
       ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME);
-      setState(521);
+      setState(522);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) {
       case 1:
         {
-        setState(519);
-        match(ON);
         setState(520);
+        match(ON);
+        setState(521);
         ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern();
         }
         break;
       }
-      setState(532);
+      setState(533);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) {
       case 1:
         {
-        setState(523);
-        match(WITH);
         setState(524);
+        match(WITH);
+        setState(525);
         enrichWithClause();
-        setState(529);
+        setState(530);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,50,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,49,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(525);
-            match(COMMA);
             setState(526);
+            match(COMMA);
+            setState(527);
             enrichWithClause();
             }
             } 
           }
-          setState(531);
+          setState(532);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,50,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,49,_ctx);
         }
         }
         break;
@@ -4785,23 +4821,23 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
 
   public final EnrichWithClauseContext enrichWithClause() throws RecognitionException {
     EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState());
-    enterRule(_localctx, 104, RULE_enrichWithClause);
+    enterRule(_localctx, 106, RULE_enrichWithClause);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(537);
+      setState(538);
       _errHandler.sync(this);
-      switch ( getInterpreter().adaptivePredict(_input,52,_ctx) ) {
+      switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) {
       case 1:
         {
-        setState(534);
-        ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern();
         setState(535);
+        ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern();
+        setState(536);
         match(ASSIGN);
         }
         break;
       }
-      setState(539);
+      setState(540);
       ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern();
       }
     }
@@ -4816,6 +4852,63 @@ public final EnrichWithClauseContext enrichWithClause() throws RecognitionExcept
     return _localctx;
   }
 
+  @SuppressWarnings("CheckReturnValue")
+  public static class LookupCommandContext extends ParserRuleContext {
+    public Token tableName;
+    public QualifiedNamePatternsContext matchFields;
+    public TerminalNode LOOKUP() { return getToken(EsqlBaseParser.LOOKUP, 0); }
+    public TerminalNode ON() { return getToken(EsqlBaseParser.ON, 0); }
+    public TerminalNode INDEX_UNQUOTED_IDENTIFIER() { return getToken(EsqlBaseParser.INDEX_UNQUOTED_IDENTIFIER, 0); }
+    public QualifiedNamePatternsContext qualifiedNamePatterns() {
+      return getRuleContext(QualifiedNamePatternsContext.class,0);
+    }
+    @SuppressWarnings("this-escape")
+    public LookupCommandContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_lookupCommand; }
+    @Override
+    public void enterRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterLookupCommand(this);
+    }
+    @Override
+    public void exitRule(ParseTreeListener listener) {
+      if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitLookupCommand(this);
+    }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor<? extends T>)visitor).visitLookupCommand(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final LookupCommandContext lookupCommand() throws RecognitionException {
+    LookupCommandContext _localctx = new LookupCommandContext(_ctx, getState());
+    enterRule(_localctx, 108, RULE_lookupCommand);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(542);
+      match(LOOKUP);
+      setState(543);
+      ((LookupCommandContext)_localctx).tableName = match(INDEX_UNQUOTED_IDENTIFIER);
+      setState(544);
+      match(ON);
+      setState(545);
+      ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns();
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
     case 1:
@@ -4863,7 +4956,7 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in
   }
 
   public static final String _serializedATN =
-    "\u0004\u0001t\u021e\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+
+    "\u0004\u0001{\u0224\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+
     "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+
     "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+
     "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+
@@ -4877,338 +4970,341 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in
     "#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002"+
     "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+
     "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+
-    "2\u00072\u00023\u00073\u00024\u00074\u0001\u0000\u0001\u0000\u0001\u0000"+
-    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+
-    "\u0005\u0001t\b\u0001\n\u0001\f\u0001w\t\u0001\u0001\u0002\u0001\u0002"+
-    "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u007f\b\u0002"+
-    "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+
-    "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+
-    "\u0001\u0003\u0003\u0003\u008e\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004"+
-    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+
-    "\u0001\u0005\u0003\u0005\u009a\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+
-    "\u0001\u0005\u0001\u0005\u0005\u0005\u00a1\b\u0005\n\u0005\f\u0005\u00a4"+
-    "\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003"+
-    "\u0005\u00ab\b\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00af\b\u0005"+
-    "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+
-    "\u0005\u0005\u00b7\b\u0005\n\u0005\f\u0005\u00ba\t\u0005\u0001\u0006\u0001"+
-    "\u0006\u0003\u0006\u00be\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+
-    "\u0006\u0001\u0006\u0003\u0006\u00c5\b\u0006\u0001\u0006\u0001\u0006\u0001"+
-    "\u0006\u0003\u0006\u00ca\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+
-    "\u0007\u0001\u0007\u0003\u0007\u00d1\b\u0007\u0001\b\u0001\b\u0001\b\u0001"+
-    "\b\u0003\b\u00d7\b\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0005"+
-    "\b\u00df\b\b\n\b\f\b\u00e2\t\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t"+
-    "\u0001\t\u0001\t\u0001\t\u0003\t\u00ec\b\t\u0001\t\u0001\t\u0001\t\u0005"+
-    "\t\u00f1\b\t\n\t\f\t\u00f4\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n"+
-    "\u0001\n\u0005\n\u00fc\b\n\n\n\f\n\u00ff\t\n\u0003\n\u0101\b\n\u0001\n"+
-    "\u0001\n\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\r\u0001"+
-    "\r\u0001\r\u0005\r\u010d\b\r\n\r\f\r\u0110\t\r\u0001\u000e\u0001\u000e"+
-    "\u0001\u000e\u0001\u000e\u0001\u000e\u0003\u000e\u0117\b\u000e\u0001\u000f"+
-    "\u0001\u000f\u0001\u000f\u0001\u000f\u0005\u000f\u011d\b\u000f\n\u000f"+
-    "\f\u000f\u0120\t\u000f\u0001\u000f\u0003\u000f\u0123\b\u000f\u0001\u0010"+
-    "\u0001\u0010\u0001\u0011\u0001\u0011\u0003\u0011\u0129\b\u0011\u0001\u0012"+
-    "\u0001\u0012\u0001\u0012\u0001\u0012\u0005\u0012\u012f\b\u0012\n\u0012"+
-    "\f\u0012\u0132\t\u0012\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+
-    "\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0005\u0014\u013c\b\u0014"+
-    "\n\u0014\f\u0014\u013f\t\u0014\u0001\u0014\u0003\u0014\u0142\b\u0014\u0001"+
-    "\u0014\u0001\u0014\u0003\u0014\u0146\b\u0014\u0001\u0015\u0001\u0015\u0001"+
-    "\u0015\u0001\u0016\u0001\u0016\u0003\u0016\u014d\b\u0016\u0001\u0016\u0001"+
-    "\u0016\u0003\u0016\u0151\b\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001"+
-    "\u0017\u0003\u0017\u0157\b\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0005"+
-    "\u0018\u015c\b\u0018\n\u0018\f\u0018\u015f\t\u0018\u0001\u0019\u0001\u0019"+
-    "\u0001\u0019\u0005\u0019\u0164\b\u0019\n\u0019\f\u0019\u0167\t\u0019\u0001"+
-    "\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001"+
-    "\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001"+
-    "\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0005\u001c\u017a"+
-    "\b\u001c\n\u001c\f\u001c\u017d\t\u001c\u0001\u001c\u0001\u001c\u0001\u001c"+
-    "\u0001\u001c\u0001\u001c\u0001\u001c\u0005\u001c\u0185\b\u001c\n\u001c"+
-    "\f\u001c\u0188\t\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c"+
-    "\u0001\u001c\u0001\u001c\u0005\u001c\u0190\b\u001c\n\u001c\f\u001c\u0193"+
-    "\t\u001c\u0001\u001c\u0001\u001c\u0003\u001c\u0197\b\u001c\u0001\u001d"+
-    "\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e"+
-    "\u0005\u001e\u01a0\b\u001e\n\u001e\f\u001e\u01a3\t\u001e\u0001\u001f\u0001"+
-    "\u001f\u0003\u001f\u01a7\b\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u01ab"+
-    "\b\u001f\u0001 \u0001 \u0001 \u0001 \u0005 \u01b1\b \n \f \u01b4\t \u0001"+
-    "!\u0001!\u0001!\u0001!\u0005!\u01ba\b!\n!\f!\u01bd\t!\u0001\"\u0001\""+
-    "\u0001\"\u0001\"\u0005\"\u01c3\b\"\n\"\f\"\u01c6\t\"\u0001#\u0001#\u0001"+
-    "#\u0001#\u0001$\u0001$\u0001$\u0001$\u0003$\u01d0\b$\u0001%\u0001%\u0001"+
-    "%\u0001%\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0005\'\u01dc\b"+
-    "\'\n\'\f\'\u01df\t\'\u0001(\u0001(\u0001(\u0001(\u0001)\u0001)\u0001*"+
-    "\u0001*\u0003*\u01e9\b*\u0001+\u0003+\u01ec\b+\u0001+\u0001+\u0001,\u0003"+
-    ",\u01f1\b,\u0001,\u0001,\u0001-\u0001-\u0001.\u0001.\u0001/\u0001/\u0001"+
-    "/\u00010\u00010\u00010\u00010\u00011\u00011\u00011\u00012\u00012\u0001"+
-    "2\u00013\u00013\u00013\u00013\u00033\u020a\b3\u00013\u00013\u00013\u0001"+
-    "3\u00053\u0210\b3\n3\f3\u0213\t3\u00033\u0215\b3\u00014\u00014\u00014"+
-    "\u00034\u021a\b4\u00014\u00014\u00014\u0000\u0004\u0002\n\u0010\u0012"+
-    "5\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a"+
-    "\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\^`bdfh\u0000\u0007\u0001"+
-    "\u0000>?\u0001\u0000@B\u0001\u0000EF\u0002\u0000\"\"&&\u0001\u0000)*\u0002"+
-    "\u0000((66\u0002\u0000779=\u023a\u0000j\u0001\u0000\u0000\u0000\u0002"+
-    "m\u0001\u0000\u0000\u0000\u0004~\u0001\u0000\u0000\u0000\u0006\u008d\u0001"+
-    "\u0000\u0000\u0000\b\u008f\u0001\u0000\u0000\u0000\n\u00ae\u0001\u0000"+
-    "\u0000\u0000\f\u00c9\u0001\u0000\u0000\u0000\u000e\u00d0\u0001\u0000\u0000"+
-    "\u0000\u0010\u00d6\u0001\u0000\u0000\u0000\u0012\u00eb\u0001\u0000\u0000"+
-    "\u0000\u0014\u00f5\u0001\u0000\u0000\u0000\u0016\u0104\u0001\u0000\u0000"+
-    "\u0000\u0018\u0106\u0001\u0000\u0000\u0000\u001a\u0109\u0001\u0000\u0000"+
-    "\u0000\u001c\u0116\u0001\u0000\u0000\u0000\u001e\u0118\u0001\u0000\u0000"+
-    "\u0000 \u0124\u0001\u0000\u0000\u0000\"\u0128\u0001\u0000\u0000\u0000"+
-    "$\u012a\u0001\u0000\u0000\u0000&\u0133\u0001\u0000\u0000\u0000(\u0137"+
-    "\u0001\u0000\u0000\u0000*\u0147\u0001\u0000\u0000\u0000,\u014a\u0001\u0000"+
-    "\u0000\u0000.\u0152\u0001\u0000\u0000\u00000\u0158\u0001\u0000\u0000\u0000"+
-    "2\u0160\u0001\u0000\u0000\u00004\u0168\u0001\u0000\u0000\u00006\u016a"+
-    "\u0001\u0000\u0000\u00008\u0196\u0001\u0000\u0000\u0000:\u0198\u0001\u0000"+
-    "\u0000\u0000<\u019b\u0001\u0000\u0000\u0000>\u01a4\u0001\u0000\u0000\u0000"+
-    "@\u01ac\u0001\u0000\u0000\u0000B\u01b5\u0001\u0000\u0000\u0000D\u01be"+
-    "\u0001\u0000\u0000\u0000F\u01c7\u0001\u0000\u0000\u0000H\u01cb\u0001\u0000"+
-    "\u0000\u0000J\u01d1\u0001\u0000\u0000\u0000L\u01d5\u0001\u0000\u0000\u0000"+
-    "N\u01d8\u0001\u0000\u0000\u0000P\u01e0\u0001\u0000\u0000\u0000R\u01e4"+
-    "\u0001\u0000\u0000\u0000T\u01e8\u0001\u0000\u0000\u0000V\u01eb\u0001\u0000"+
-    "\u0000\u0000X\u01f0\u0001\u0000\u0000\u0000Z\u01f4\u0001\u0000\u0000\u0000"+
-    "\\\u01f6\u0001\u0000\u0000\u0000^\u01f8\u0001\u0000\u0000\u0000`\u01fb"+
-    "\u0001\u0000\u0000\u0000b\u01ff\u0001\u0000\u0000\u0000d\u0202\u0001\u0000"+
-    "\u0000\u0000f\u0205\u0001\u0000\u0000\u0000h\u0219\u0001\u0000\u0000\u0000"+
-    "jk\u0003\u0002\u0001\u0000kl\u0005\u0000\u0000\u0001l\u0001\u0001\u0000"+
-    "\u0000\u0000mn\u0006\u0001\uffff\uffff\u0000no\u0003\u0004\u0002\u0000"+
-    "ou\u0001\u0000\u0000\u0000pq\n\u0001\u0000\u0000qr\u0005\u001c\u0000\u0000"+
-    "rt\u0003\u0006\u0003\u0000sp\u0001\u0000\u0000\u0000tw\u0001\u0000\u0000"+
-    "\u0000us\u0001\u0000\u0000\u0000uv\u0001\u0000\u0000\u0000v\u0003\u0001"+
-    "\u0000\u0000\u0000wu\u0001\u0000\u0000\u0000x\u007f\u0003^/\u0000y\u007f"+
-    "\u0003\u001e\u000f\u0000z\u007f\u0003\u0018\f\u0000{\u007f\u0003(\u0014"+
-    "\u0000|\u007f\u0003b1\u0000}\u007f\u0003d2\u0000~x\u0001\u0000\u0000\u0000"+
-    "~y\u0001\u0000\u0000\u0000~z\u0001\u0000\u0000\u0000~{\u0001\u0000\u0000"+
-    "\u0000~|\u0001\u0000\u0000\u0000~}\u0001\u0000\u0000\u0000\u007f\u0005"+
-    "\u0001\u0000\u0000\u0000\u0080\u008e\u0003*\u0015\u0000\u0081\u008e\u0003"+
-    ".\u0017\u0000\u0082\u008e\u0003:\u001d\u0000\u0083\u008e\u0003@ \u0000"+
-    "\u0084\u008e\u0003<\u001e\u0000\u0085\u008e\u0003,\u0016\u0000\u0086\u008e"+
-    "\u0003\b\u0004\u0000\u0087\u008e\u0003B!\u0000\u0088\u008e\u0003D\"\u0000"+
-    "\u0089\u008e\u0003H$\u0000\u008a\u008e\u0003J%\u0000\u008b\u008e\u0003"+
-    "f3\u0000\u008c\u008e\u0003L&\u0000\u008d\u0080\u0001\u0000\u0000\u0000"+
-    "\u008d\u0081\u0001\u0000\u0000\u0000\u008d\u0082\u0001\u0000\u0000\u0000"+
-    "\u008d\u0083\u0001\u0000\u0000\u0000\u008d\u0084\u0001\u0000\u0000\u0000"+
-    "\u008d\u0085\u0001\u0000\u0000\u0000\u008d\u0086\u0001\u0000\u0000\u0000"+
-    "\u008d\u0087\u0001\u0000\u0000\u0000\u008d\u0088\u0001\u0000\u0000\u0000"+
-    "\u008d\u0089\u0001\u0000\u0000\u0000\u008d\u008a\u0001\u0000\u0000\u0000"+
-    "\u008d\u008b\u0001\u0000\u0000\u0000\u008d\u008c\u0001\u0000\u0000\u0000"+
-    "\u008e\u0007\u0001\u0000\u0000\u0000\u008f\u0090\u0005\u0013\u0000\u0000"+
-    "\u0090\u0091\u0003\n\u0005\u0000\u0091\t\u0001\u0000\u0000\u0000\u0092"+
-    "\u0093\u0006\u0005\uffff\uffff\u0000\u0093\u0094\u0005/\u0000\u0000\u0094"+
-    "\u00af\u0003\n\u0005\u0007\u0095\u00af\u0003\u000e\u0007\u0000\u0096\u00af"+
-    "\u0003\f\u0006\u0000\u0097\u0099\u0003\u000e\u0007\u0000\u0098\u009a\u0005"+
-    "/\u0000\u0000\u0099\u0098\u0001\u0000\u0000\u0000\u0099\u009a\u0001\u0000"+
-    "\u0000\u0000\u009a\u009b\u0001\u0000\u0000\u0000\u009b\u009c\u0005,\u0000"+
-    "\u0000\u009c\u009d\u0005+\u0000\u0000\u009d\u00a2\u0003\u000e\u0007\u0000"+
-    "\u009e\u009f\u0005%\u0000\u0000\u009f\u00a1\u0003\u000e\u0007\u0000\u00a0"+
-    "\u009e\u0001\u0000\u0000\u0000\u00a1\u00a4\u0001\u0000\u0000\u0000\u00a2"+
-    "\u00a0\u0001\u0000\u0000\u0000\u00a2\u00a3\u0001\u0000\u0000\u0000\u00a3"+
-    "\u00a5\u0001\u0000\u0000\u0000\u00a4\u00a2\u0001\u0000\u0000\u0000\u00a5"+
-    "\u00a6\u00055\u0000\u0000\u00a6\u00af\u0001\u0000\u0000\u0000\u00a7\u00a8"+
-    "\u0003\u000e\u0007\u0000\u00a8\u00aa\u0005-\u0000\u0000\u00a9\u00ab\u0005"+
-    "/\u0000\u0000\u00aa\u00a9\u0001\u0000\u0000\u0000\u00aa\u00ab\u0001\u0000"+
-    "\u0000\u0000\u00ab\u00ac\u0001\u0000\u0000\u0000\u00ac\u00ad\u00050\u0000"+
-    "\u0000\u00ad\u00af\u0001\u0000\u0000\u0000\u00ae\u0092\u0001\u0000\u0000"+
-    "\u0000\u00ae\u0095\u0001\u0000\u0000\u0000\u00ae\u0096\u0001\u0000\u0000"+
-    "\u0000\u00ae\u0097\u0001\u0000\u0000\u0000\u00ae\u00a7\u0001\u0000\u0000"+
-    "\u0000\u00af\u00b8\u0001\u0000\u0000\u0000\u00b0\u00b1\n\u0004\u0000\u0000"+
-    "\u00b1\u00b2\u0005!\u0000\u0000\u00b2\u00b7\u0003\n\u0005\u0005\u00b3"+
-    "\u00b4\n\u0003\u0000\u0000\u00b4\u00b5\u00052\u0000\u0000\u00b5\u00b7"+
-    "\u0003\n\u0005\u0004\u00b6\u00b0\u0001\u0000\u0000\u0000\u00b6\u00b3\u0001"+
-    "\u0000\u0000\u0000\u00b7\u00ba\u0001\u0000\u0000\u0000\u00b8\u00b6\u0001"+
-    "\u0000\u0000\u0000\u00b8\u00b9\u0001\u0000\u0000\u0000\u00b9\u000b\u0001"+
-    "\u0000\u0000\u0000\u00ba\u00b8\u0001\u0000\u0000\u0000\u00bb\u00bd\u0003"+
-    "\u000e\u0007\u0000\u00bc\u00be\u0005/\u0000\u0000\u00bd\u00bc\u0001\u0000"+
-    "\u0000\u0000\u00bd\u00be\u0001\u0000\u0000\u0000\u00be\u00bf\u0001\u0000"+
-    "\u0000\u0000\u00bf\u00c0\u0005.\u0000\u0000\u00c0\u00c1\u0003Z-\u0000"+
-    "\u00c1\u00ca\u0001\u0000\u0000\u0000\u00c2\u00c4\u0003\u000e\u0007\u0000"+
-    "\u00c3\u00c5\u0005/\u0000\u0000\u00c4\u00c3\u0001\u0000\u0000\u0000\u00c4"+
-    "\u00c5\u0001\u0000\u0000\u0000\u00c5\u00c6\u0001\u0000\u0000\u0000\u00c6"+
-    "\u00c7\u00054\u0000\u0000\u00c7\u00c8\u0003Z-\u0000\u00c8\u00ca\u0001"+
-    "\u0000\u0000\u0000\u00c9\u00bb\u0001\u0000\u0000\u0000\u00c9\u00c2\u0001"+
-    "\u0000\u0000\u0000\u00ca\r\u0001\u0000\u0000\u0000\u00cb\u00d1\u0003\u0010"+
-    "\b\u0000\u00cc\u00cd\u0003\u0010\b\u0000\u00cd\u00ce\u0003\\.\u0000\u00ce"+
-    "\u00cf\u0003\u0010\b\u0000\u00cf\u00d1\u0001\u0000\u0000\u0000\u00d0\u00cb"+
-    "\u0001\u0000\u0000\u0000\u00d0\u00cc\u0001\u0000\u0000\u0000\u00d1\u000f"+
-    "\u0001\u0000\u0000\u0000\u00d2\u00d3\u0006\b\uffff\uffff\u0000\u00d3\u00d7"+
-    "\u0003\u0012\t\u0000\u00d4\u00d5\u0007\u0000\u0000\u0000\u00d5\u00d7\u0003"+
-    "\u0010\b\u0003\u00d6\u00d2\u0001\u0000\u0000\u0000\u00d6\u00d4\u0001\u0000"+
-    "\u0000\u0000\u00d7\u00e0\u0001\u0000\u0000\u0000\u00d8\u00d9\n\u0002\u0000"+
-    "\u0000\u00d9\u00da\u0007\u0001\u0000\u0000\u00da\u00df\u0003\u0010\b\u0003"+
-    "\u00db\u00dc\n\u0001\u0000\u0000\u00dc\u00dd\u0007\u0000\u0000\u0000\u00dd"+
-    "\u00df\u0003\u0010\b\u0002\u00de\u00d8\u0001\u0000\u0000\u0000\u00de\u00db"+
-    "\u0001\u0000\u0000\u0000\u00df\u00e2\u0001\u0000\u0000\u0000\u00e0\u00de"+
-    "\u0001\u0000\u0000\u0000\u00e0\u00e1\u0001\u0000\u0000\u0000\u00e1\u0011"+
-    "\u0001\u0000\u0000\u0000\u00e2\u00e0\u0001\u0000\u0000\u0000\u00e3\u00e4"+
-    "\u0006\t\uffff\uffff\u0000\u00e4\u00ec\u00038\u001c\u0000\u00e5\u00ec"+
-    "\u00030\u0018\u0000\u00e6\u00ec\u0003\u0014\n\u0000\u00e7\u00e8\u0005"+
-    "+\u0000\u0000\u00e8\u00e9\u0003\n\u0005\u0000\u00e9\u00ea\u00055\u0000"+
-    "\u0000\u00ea\u00ec\u0001\u0000\u0000\u0000\u00eb\u00e3\u0001\u0000\u0000"+
-    "\u0000\u00eb\u00e5\u0001\u0000\u0000\u0000\u00eb\u00e6\u0001\u0000\u0000"+
-    "\u0000\u00eb\u00e7\u0001\u0000\u0000\u0000\u00ec\u00f2\u0001\u0000\u0000"+
-    "\u0000\u00ed\u00ee\n\u0001\u0000\u0000\u00ee\u00ef\u0005$\u0000\u0000"+
-    "\u00ef\u00f1\u0003\u0016\u000b\u0000\u00f0\u00ed\u0001\u0000\u0000\u0000"+
-    "\u00f1\u00f4\u0001\u0000\u0000\u0000\u00f2\u00f0\u0001\u0000\u0000\u0000"+
-    "\u00f2\u00f3\u0001\u0000\u0000\u0000\u00f3\u0013\u0001\u0000\u0000\u0000"+
-    "\u00f4\u00f2\u0001\u0000\u0000\u0000\u00f5\u00f6\u00034\u001a\u0000\u00f6"+
-    "\u0100\u0005+\u0000\u0000\u00f7\u0101\u0005@\u0000\u0000\u00f8\u00fd\u0003"+
-    "\n\u0005\u0000\u00f9\u00fa\u0005%\u0000\u0000\u00fa\u00fc\u0003\n\u0005"+
-    "\u0000\u00fb\u00f9\u0001\u0000\u0000\u0000\u00fc\u00ff\u0001\u0000\u0000"+
-    "\u0000\u00fd\u00fb\u0001\u0000\u0000\u0000\u00fd\u00fe\u0001\u0000\u0000"+
-    "\u0000\u00fe\u0101\u0001\u0000\u0000\u0000\u00ff\u00fd\u0001\u0000\u0000"+
-    "\u0000\u0100\u00f7\u0001\u0000\u0000\u0000\u0100\u00f8\u0001\u0000\u0000"+
-    "\u0000\u0100\u0101\u0001\u0000\u0000\u0000\u0101\u0102\u0001\u0000\u0000"+
-    "\u0000\u0102\u0103\u00055\u0000\u0000\u0103\u0015\u0001\u0000\u0000\u0000"+
-    "\u0104\u0105\u00034\u001a\u0000\u0105\u0017\u0001\u0000\u0000\u0000\u0106"+
-    "\u0107\u0005\u000f\u0000\u0000\u0107\u0108\u0003\u001a\r\u0000\u0108\u0019"+
-    "\u0001\u0000\u0000\u0000\u0109\u010e\u0003\u001c\u000e\u0000\u010a\u010b"+
-    "\u0005%\u0000\u0000\u010b\u010d\u0003\u001c\u000e\u0000\u010c\u010a\u0001"+
-    "\u0000\u0000\u0000\u010d\u0110\u0001\u0000\u0000\u0000\u010e\u010c\u0001"+
-    "\u0000\u0000\u0000\u010e\u010f\u0001\u0000\u0000\u0000\u010f\u001b\u0001"+
-    "\u0000\u0000\u0000\u0110\u010e\u0001\u0000\u0000\u0000\u0111\u0117\u0003"+
-    "\n\u0005\u0000\u0112\u0113\u00030\u0018\u0000\u0113\u0114\u0005#\u0000"+
-    "\u0000\u0114\u0115\u0003\n\u0005\u0000\u0115\u0117\u0001\u0000\u0000\u0000"+
-    "\u0116\u0111\u0001\u0000\u0000\u0000\u0116\u0112\u0001\u0000\u0000\u0000"+
-    "\u0117\u001d\u0001\u0000\u0000\u0000\u0118\u0119\u0005\u0006\u0000\u0000"+
-    "\u0119\u011e\u0003 \u0010\u0000\u011a\u011b\u0005%\u0000\u0000\u011b\u011d"+
-    "\u0003 \u0010\u0000\u011c\u011a\u0001\u0000\u0000\u0000\u011d\u0120\u0001"+
-    "\u0000\u0000\u0000\u011e\u011c\u0001\u0000\u0000\u0000\u011e\u011f\u0001"+
-    "\u0000\u0000\u0000\u011f\u0122\u0001\u0000\u0000\u0000\u0120\u011e\u0001"+
-    "\u0000\u0000\u0000\u0121\u0123\u0003\"\u0011\u0000\u0122\u0121\u0001\u0000"+
-    "\u0000\u0000\u0122\u0123\u0001\u0000\u0000\u0000\u0123\u001f\u0001\u0000"+
-    "\u0000\u0000\u0124\u0125\u0005\u0018\u0000\u0000\u0125!\u0001\u0000\u0000"+
-    "\u0000\u0126\u0129\u0003$\u0012\u0000\u0127\u0129\u0003&\u0013\u0000\u0128"+
-    "\u0126\u0001\u0000\u0000\u0000\u0128\u0127\u0001\u0000\u0000\u0000\u0129"+
-    "#\u0001\u0000\u0000\u0000\u012a\u012b\u0005J\u0000\u0000\u012b\u0130\u0003"+
-    " \u0010\u0000\u012c\u012d\u0005%\u0000\u0000\u012d\u012f\u0003 \u0010"+
-    "\u0000\u012e\u012c\u0001\u0000\u0000\u0000\u012f\u0132\u0001\u0000\u0000"+
-    "\u0000\u0130\u012e\u0001\u0000\u0000\u0000\u0130\u0131\u0001\u0000\u0000"+
-    "\u0000\u0131%\u0001\u0000\u0000\u0000\u0132\u0130\u0001\u0000\u0000\u0000"+
-    "\u0133\u0134\u0005C\u0000\u0000\u0134\u0135\u0003$\u0012\u0000\u0135\u0136"+
-    "\u0005D\u0000\u0000\u0136\'\u0001\u0000\u0000\u0000\u0137\u0138\u0005"+
-    "\f\u0000\u0000\u0138\u013d\u0003 \u0010\u0000\u0139\u013a\u0005%\u0000"+
-    "\u0000\u013a\u013c\u0003 \u0010\u0000\u013b\u0139\u0001\u0000\u0000\u0000"+
-    "\u013c\u013f\u0001\u0000\u0000\u0000\u013d\u013b\u0001\u0000\u0000\u0000"+
-    "\u013d\u013e\u0001\u0000\u0000\u0000\u013e\u0141\u0001\u0000\u0000\u0000"+
-    "\u013f\u013d\u0001\u0000\u0000\u0000\u0140\u0142\u0003\u001a\r\u0000\u0141"+
-    "\u0140\u0001\u0000\u0000\u0000\u0141\u0142\u0001\u0000\u0000\u0000\u0142"+
-    "\u0145\u0001\u0000\u0000\u0000\u0143\u0144\u0005 \u0000\u0000\u0144\u0146"+
-    "\u0003\u001a\r\u0000\u0145\u0143\u0001\u0000\u0000\u0000\u0145\u0146\u0001"+
-    "\u0000\u0000\u0000\u0146)\u0001\u0000\u0000\u0000\u0147\u0148\u0005\u0004"+
-    "\u0000\u0000\u0148\u0149\u0003\u001a\r\u0000\u0149+\u0001\u0000\u0000"+
-    "\u0000\u014a\u014c\u0005\u0012\u0000\u0000\u014b\u014d\u0003\u001a\r\u0000"+
-    "\u014c\u014b\u0001\u0000\u0000\u0000\u014c\u014d\u0001\u0000\u0000\u0000"+
-    "\u014d\u0150\u0001\u0000\u0000\u0000\u014e\u014f\u0005 \u0000\u0000\u014f"+
-    "\u0151\u0003\u001a\r\u0000\u0150\u014e\u0001\u0000\u0000\u0000\u0150\u0151"+
-    "\u0001\u0000\u0000\u0000\u0151-\u0001\u0000\u0000\u0000\u0152\u0153\u0005"+
-    "\b\u0000\u0000\u0153\u0156\u0003\u001a\r\u0000\u0154\u0155\u0005 \u0000"+
-    "\u0000\u0155\u0157\u0003\u001a\r\u0000\u0156\u0154\u0001\u0000\u0000\u0000"+
-    "\u0156\u0157\u0001\u0000\u0000\u0000\u0157/\u0001\u0000\u0000\u0000\u0158"+
-    "\u015d\u00034\u001a\u0000\u0159\u015a\u0005\'\u0000\u0000\u015a\u015c"+
-    "\u00034\u001a\u0000\u015b\u0159\u0001\u0000\u0000\u0000\u015c\u015f\u0001"+
-    "\u0000\u0000\u0000\u015d\u015b\u0001\u0000\u0000\u0000\u015d\u015e\u0001"+
-    "\u0000\u0000\u0000\u015e1\u0001\u0000\u0000\u0000\u015f\u015d\u0001\u0000"+
-    "\u0000\u0000\u0160\u0165\u00036\u001b\u0000\u0161\u0162\u0005\'\u0000"+
-    "\u0000\u0162\u0164\u00036\u001b\u0000\u0163\u0161\u0001\u0000\u0000\u0000"+
-    "\u0164\u0167\u0001\u0000\u0000\u0000\u0165\u0163\u0001\u0000\u0000\u0000"+
-    "\u0165\u0166\u0001\u0000\u0000\u0000\u01663\u0001\u0000\u0000\u0000\u0167"+
-    "\u0165\u0001\u0000\u0000\u0000\u0168\u0169\u0007\u0002\u0000\u0000\u0169"+
-    "5\u0001\u0000\u0000\u0000\u016a\u016b\u0005N\u0000\u0000\u016b7\u0001"+
-    "\u0000\u0000\u0000\u016c\u0197\u00050\u0000\u0000\u016d\u016e\u0003X,"+
-    "\u0000\u016e\u016f\u0005E\u0000\u0000\u016f\u0197\u0001\u0000\u0000\u0000"+
-    "\u0170\u0197\u0003V+\u0000\u0171\u0197\u0003X,\u0000\u0172\u0197\u0003"+
-    "R)\u0000\u0173\u0197\u00053\u0000\u0000\u0174\u0197\u0003Z-\u0000\u0175"+
-    "\u0176\u0005C\u0000\u0000\u0176\u017b\u0003T*\u0000\u0177\u0178\u0005"+
-    "%\u0000\u0000\u0178\u017a\u0003T*\u0000\u0179\u0177\u0001\u0000\u0000"+
-    "\u0000\u017a\u017d\u0001\u0000\u0000\u0000\u017b\u0179\u0001\u0000\u0000"+
-    "\u0000\u017b\u017c\u0001\u0000\u0000\u0000\u017c\u017e\u0001\u0000\u0000"+
-    "\u0000\u017d\u017b\u0001\u0000\u0000\u0000\u017e\u017f\u0005D\u0000\u0000"+
-    "\u017f\u0197\u0001\u0000\u0000\u0000\u0180\u0181\u0005C\u0000\u0000\u0181"+
-    "\u0186\u0003R)\u0000\u0182\u0183\u0005%\u0000\u0000\u0183\u0185\u0003"+
-    "R)\u0000\u0184\u0182\u0001\u0000\u0000\u0000\u0185\u0188\u0001\u0000\u0000"+
-    "\u0000\u0186\u0184\u0001\u0000\u0000\u0000\u0186\u0187\u0001\u0000\u0000"+
-    "\u0000\u0187\u0189\u0001\u0000\u0000\u0000\u0188\u0186\u0001\u0000\u0000"+
-    "\u0000\u0189\u018a\u0005D\u0000\u0000\u018a\u0197\u0001\u0000\u0000\u0000"+
-    "\u018b\u018c\u0005C\u0000\u0000\u018c\u0191\u0003Z-\u0000\u018d\u018e"+
-    "\u0005%\u0000\u0000\u018e\u0190\u0003Z-\u0000\u018f\u018d\u0001\u0000"+
-    "\u0000\u0000\u0190\u0193\u0001\u0000\u0000\u0000\u0191\u018f\u0001\u0000"+
-    "\u0000\u0000\u0191\u0192\u0001\u0000\u0000\u0000\u0192\u0194\u0001\u0000"+
-    "\u0000\u0000\u0193\u0191\u0001\u0000\u0000\u0000\u0194\u0195\u0005D\u0000"+
-    "\u0000\u0195\u0197\u0001\u0000\u0000\u0000\u0196\u016c\u0001\u0000\u0000"+
-    "\u0000\u0196\u016d\u0001\u0000\u0000\u0000\u0196\u0170\u0001\u0000\u0000"+
-    "\u0000\u0196\u0171\u0001\u0000\u0000\u0000\u0196\u0172\u0001\u0000\u0000"+
-    "\u0000\u0196\u0173\u0001\u0000\u0000\u0000\u0196\u0174\u0001\u0000\u0000"+
-    "\u0000\u0196\u0175\u0001\u0000\u0000\u0000\u0196\u0180\u0001\u0000\u0000"+
-    "\u0000\u0196\u018b\u0001\u0000\u0000\u0000\u01979\u0001\u0000\u0000\u0000"+
-    "\u0198\u0199\u0005\n\u0000\u0000\u0199\u019a\u0005\u001e\u0000\u0000\u019a"+
-    ";\u0001\u0000\u0000\u0000\u019b\u019c\u0005\u0011\u0000\u0000\u019c\u01a1"+
-    "\u0003>\u001f\u0000\u019d\u019e\u0005%\u0000\u0000\u019e\u01a0\u0003>"+
-    "\u001f\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u01a0\u01a3\u0001\u0000"+
-    "\u0000\u0000\u01a1\u019f\u0001\u0000\u0000\u0000\u01a1\u01a2\u0001\u0000"+
-    "\u0000\u0000\u01a2=\u0001\u0000\u0000\u0000\u01a3\u01a1\u0001\u0000\u0000"+
-    "\u0000\u01a4\u01a6\u0003\n\u0005\u0000\u01a5\u01a7\u0007\u0003\u0000\u0000"+
-    "\u01a6\u01a5\u0001\u0000\u0000\u0000\u01a6\u01a7\u0001\u0000\u0000\u0000"+
-    "\u01a7\u01aa\u0001\u0000\u0000\u0000\u01a8\u01a9\u00051\u0000\u0000\u01a9"+
-    "\u01ab\u0007\u0004\u0000\u0000\u01aa\u01a8\u0001\u0000\u0000\u0000\u01aa"+
-    "\u01ab\u0001\u0000\u0000\u0000\u01ab?\u0001\u0000\u0000\u0000\u01ac\u01ad"+
-    "\u0005\t\u0000\u0000\u01ad\u01b2\u00032\u0019\u0000\u01ae\u01af\u0005"+
-    "%\u0000\u0000\u01af\u01b1\u00032\u0019\u0000\u01b0\u01ae\u0001\u0000\u0000"+
-    "\u0000\u01b1\u01b4\u0001\u0000\u0000\u0000\u01b2\u01b0\u0001\u0000\u0000"+
-    "\u0000\u01b2\u01b3\u0001\u0000\u0000\u0000\u01b3A\u0001\u0000\u0000\u0000"+
-    "\u01b4\u01b2\u0001\u0000\u0000\u0000\u01b5\u01b6\u0005\u0002\u0000\u0000"+
-    "\u01b6\u01bb\u00032\u0019\u0000\u01b7\u01b8\u0005%\u0000\u0000\u01b8\u01ba"+
-    "\u00032\u0019\u0000\u01b9\u01b7\u0001\u0000\u0000\u0000\u01ba\u01bd\u0001"+
-    "\u0000\u0000\u0000\u01bb\u01b9\u0001\u0000\u0000\u0000\u01bb\u01bc\u0001"+
-    "\u0000\u0000\u0000\u01bcC\u0001\u0000\u0000\u0000\u01bd\u01bb\u0001\u0000"+
-    "\u0000\u0000\u01be\u01bf\u0005\u000e\u0000\u0000\u01bf\u01c4\u0003F#\u0000"+
-    "\u01c0\u01c1\u0005%\u0000\u0000\u01c1\u01c3\u0003F#\u0000\u01c2\u01c0"+
-    "\u0001\u0000\u0000\u0000\u01c3\u01c6\u0001\u0000\u0000\u0000\u01c4\u01c2"+
-    "\u0001\u0000\u0000\u0000\u01c4\u01c5\u0001\u0000\u0000\u0000\u01c5E\u0001"+
-    "\u0000\u0000\u0000\u01c6\u01c4\u0001\u0000\u0000\u0000\u01c7\u01c8\u0003"+
-    "2\u0019\u0000\u01c8\u01c9\u0005R\u0000\u0000\u01c9\u01ca\u00032\u0019"+
-    "\u0000\u01caG\u0001\u0000\u0000\u0000\u01cb\u01cc\u0005\u0001\u0000\u0000"+
-    "\u01cc\u01cd\u0003\u0012\t\u0000\u01cd\u01cf\u0003Z-\u0000\u01ce\u01d0"+
-    "\u0003N\'\u0000\u01cf\u01ce\u0001\u0000\u0000\u0000\u01cf\u01d0\u0001"+
-    "\u0000\u0000\u0000\u01d0I\u0001\u0000\u0000\u0000\u01d1\u01d2\u0005\u0007"+
-    "\u0000\u0000\u01d2\u01d3\u0003\u0012\t\u0000\u01d3\u01d4\u0003Z-\u0000"+
-    "\u01d4K\u0001\u0000\u0000\u0000\u01d5\u01d6\u0005\r\u0000\u0000\u01d6"+
-    "\u01d7\u00030\u0018\u0000\u01d7M\u0001\u0000\u0000\u0000\u01d8\u01dd\u0003"+
-    "P(\u0000\u01d9\u01da\u0005%\u0000\u0000\u01da\u01dc\u0003P(\u0000\u01db"+
-    "\u01d9\u0001\u0000\u0000\u0000\u01dc\u01df\u0001\u0000\u0000\u0000\u01dd"+
-    "\u01db\u0001\u0000\u0000\u0000\u01dd\u01de\u0001\u0000\u0000\u0000\u01de"+
-    "O\u0001\u0000\u0000\u0000\u01df\u01dd\u0001\u0000\u0000\u0000\u01e0\u01e1"+
-    "\u00034\u001a\u0000\u01e1\u01e2\u0005#\u0000\u0000\u01e2\u01e3\u00038"+
-    "\u001c\u0000\u01e3Q\u0001\u0000\u0000\u0000\u01e4\u01e5\u0007\u0005\u0000"+
-    "\u0000\u01e5S\u0001\u0000\u0000\u0000\u01e6\u01e9\u0003V+\u0000\u01e7"+
-    "\u01e9\u0003X,\u0000\u01e8\u01e6\u0001\u0000\u0000\u0000\u01e8\u01e7\u0001"+
-    "\u0000\u0000\u0000\u01e9U\u0001\u0000\u0000\u0000\u01ea\u01ec\u0007\u0000"+
-    "\u0000\u0000\u01eb\u01ea\u0001\u0000\u0000\u0000\u01eb\u01ec\u0001\u0000"+
-    "\u0000\u0000\u01ec\u01ed\u0001\u0000\u0000\u0000\u01ed\u01ee\u0005\u001f"+
-    "\u0000\u0000\u01eeW\u0001\u0000\u0000\u0000\u01ef\u01f1\u0007\u0000\u0000"+
-    "\u0000\u01f0\u01ef\u0001\u0000\u0000\u0000\u01f0\u01f1\u0001\u0000\u0000"+
-    "\u0000\u01f1\u01f2\u0001\u0000\u0000\u0000\u01f2\u01f3\u0005\u001e\u0000"+
-    "\u0000\u01f3Y\u0001\u0000\u0000\u0000\u01f4\u01f5\u0005\u001d\u0000\u0000"+
-    "\u01f5[\u0001\u0000\u0000\u0000\u01f6\u01f7\u0007\u0006\u0000\u0000\u01f7"+
-    "]\u0001\u0000\u0000\u0000\u01f8\u01f9\u0005\u0005\u0000\u0000\u01f9\u01fa"+
-    "\u0003`0\u0000\u01fa_\u0001\u0000\u0000\u0000\u01fb\u01fc\u0005C\u0000"+
-    "\u0000\u01fc\u01fd\u0003\u0002\u0001\u0000\u01fd\u01fe\u0005D\u0000\u0000"+
-    "\u01fea\u0001\u0000\u0000\u0000\u01ff\u0200\u0005\u0010\u0000\u0000\u0200"+
-    "\u0201\u0005b\u0000\u0000\u0201c\u0001\u0000\u0000\u0000\u0202\u0203\u0005"+
-    "\u000b\u0000\u0000\u0203\u0204\u0005f\u0000\u0000\u0204e\u0001\u0000\u0000"+
-    "\u0000\u0205\u0206\u0005\u0003\u0000\u0000\u0206\u0209\u0005X\u0000\u0000"+
-    "\u0207\u0208\u0005V\u0000\u0000\u0208\u020a\u00032\u0019\u0000\u0209\u0207"+
-    "\u0001\u0000\u0000\u0000\u0209\u020a\u0001\u0000\u0000\u0000\u020a\u0214"+
-    "\u0001\u0000\u0000\u0000\u020b\u020c\u0005W\u0000\u0000\u020c\u0211\u0003"+
-    "h4\u0000\u020d\u020e\u0005%\u0000\u0000\u020e\u0210\u0003h4\u0000\u020f"+
-    "\u020d\u0001\u0000\u0000\u0000\u0210\u0213\u0001\u0000\u0000\u0000\u0211"+
-    "\u020f\u0001\u0000\u0000\u0000\u0211\u0212\u0001\u0000\u0000\u0000\u0212"+
-    "\u0215\u0001\u0000\u0000\u0000\u0213\u0211\u0001\u0000\u0000\u0000\u0214"+
-    "\u020b\u0001\u0000\u0000\u0000\u0214\u0215\u0001\u0000\u0000\u0000\u0215"+
-    "g\u0001\u0000\u0000\u0000\u0216\u0217\u00032\u0019\u0000\u0217\u0218\u0005"+
-    "#\u0000\u0000\u0218\u021a\u0001\u0000\u0000\u0000\u0219\u0216\u0001\u0000"+
-    "\u0000\u0000\u0219\u021a\u0001\u0000\u0000\u0000\u021a\u021b\u0001\u0000"+
-    "\u0000\u0000\u021b\u021c\u00032\u0019\u0000\u021ci\u0001\u0000\u0000\u0000"+
-    "5u~\u008d\u0099\u00a2\u00aa\u00ae\u00b6\u00b8\u00bd\u00c4\u00c9\u00d0"+
-    "\u00d6\u00de\u00e0\u00eb\u00f2\u00fd\u0100\u010e\u0116\u011e\u0122\u0128"+
-    "\u0130\u013d\u0141\u0145\u014c\u0150\u0156\u015d\u0165\u017b\u0186\u0191"+
-    "\u0196\u01a1\u01a6\u01aa\u01b2\u01bb\u01c4\u01cf\u01dd\u01e8\u01eb\u01f0"+
-    "\u0209\u0211\u0214\u0219";
+    "2\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u0001"+
+    "\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+
+    "\u0001\u0001\u0001\u0001\u0001\u0005\u0001x\b\u0001\n\u0001\f\u0001{\t"+
+    "\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+
+    "\u0002\u0003\u0002\u0083\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
+    "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+
+    "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u0093"+
+    "\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001"+
+    "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u009f"+
+    "\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005"+
+    "\u0005\u00a6\b\u0005\n\u0005\f\u0005\u00a9\t\u0005\u0001\u0005\u0001\u0005"+
+    "\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00b0\b\u0005\u0001\u0005"+
+    "\u0001\u0005\u0003\u0005\u00b4\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+
+    "\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00bc\b\u0005\n\u0005"+
+    "\f\u0005\u00bf\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006\u00c3\b\u0006"+
+    "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006"+
+    "\u00ca\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00cf\b"+
+    "\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0003"+
+    "\u0007\u00d6\b\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00dc\b\b"+
+    "\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0005\b\u00e4\b\b\n\b"+
+    "\f\b\u00e7\t\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t"+
+    "\u0001\t\u0003\t\u00f1\b\t\u0001\t\u0001\t\u0001\t\u0005\t\u00f6\b\t\n"+
+    "\t\f\t\u00f9\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0005"+
+    "\n\u0101\b\n\n\n\f\n\u0104\t\n\u0003\n\u0106\b\n\u0001\n\u0001\n\u0001"+
+    "\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0005"+
+    "\r\u0112\b\r\n\r\f\r\u0115\t\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001"+
+    "\u000e\u0001\u000e\u0003\u000e\u011c\b\u000e\u0001\u000f\u0001\u000f\u0001"+
+    "\u000f\u0001\u000f\u0005\u000f\u0122\b\u000f\n\u000f\f\u000f\u0125\t\u000f"+
+    "\u0001\u000f\u0003\u000f\u0128\b\u000f\u0001\u0010\u0001\u0010\u0001\u0011"+
+    "\u0001\u0011\u0003\u0011\u012e\b\u0011\u0001\u0012\u0001\u0012\u0001\u0012"+
+    "\u0001\u0012\u0005\u0012\u0134\b\u0012\n\u0012\f\u0012\u0137\t\u0012\u0001"+
+    "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001"+
+    "\u0014\u0001\u0014\u0005\u0014\u0141\b\u0014\n\u0014\f\u0014\u0144\t\u0014"+
+    "\u0001\u0014\u0003\u0014\u0147\b\u0014\u0001\u0014\u0001\u0014\u0003\u0014"+
+    "\u014b\b\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016"+
+    "\u0003\u0016\u0152\b\u0016\u0001\u0016\u0001\u0016\u0003\u0016\u0156\b"+
+    "\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0003\u0017\u015c"+
+    "\b\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0005\u0018\u0161\b\u0018"+
+    "\n\u0018\f\u0018\u0164\t\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0005"+
+    "\u0019\u0169\b\u0019\n\u0019\f\u0019\u016c\t\u0019\u0001\u001a\u0001\u001a"+
+    "\u0001\u001a\u0005\u001a\u0171\b\u001a\n\u001a\f\u001a\u0174\t\u001a\u0001"+
+    "\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d\u0001"+
+    "\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0001"+
+    "\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u0187"+
+    "\b\u001d\n\u001d\f\u001d\u018a\t\u001d\u0001\u001d\u0001\u001d\u0001\u001d"+
+    "\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u0192\b\u001d\n\u001d"+
+    "\f\u001d\u0195\t\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d"+
+    "\u0001\u001d\u0001\u001d\u0005\u001d\u019d\b\u001d\n\u001d\f\u001d\u01a0"+
+    "\t\u001d\u0001\u001d\u0001\u001d\u0003\u001d\u01a4\b\u001d\u0001\u001e"+
+    "\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+
+    "\u0005\u001f\u01ad\b\u001f\n\u001f\f\u001f\u01b0\t\u001f\u0001 \u0001"+
+    " \u0003 \u01b4\b \u0001 \u0001 \u0003 \u01b8\b \u0001!\u0001!\u0001!\u0001"+
+    "\"\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0001#\u0005#\u01c4\b#\n#\f#\u01c7"+
+    "\t#\u0001$\u0001$\u0001$\u0001$\u0001%\u0001%\u0001%\u0001%\u0003%\u01d1"+
+    "\b%\u0001&\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001(\u0001("+
+    "\u0001(\u0005(\u01dd\b(\n(\f(\u01e0\t(\u0001)\u0001)\u0001)\u0001)\u0001"+
+    "*\u0001*\u0001+\u0001+\u0003+\u01ea\b+\u0001,\u0003,\u01ed\b,\u0001,\u0001"+
+    ",\u0001-\u0003-\u01f2\b-\u0001-\u0001-\u0001.\u0001.\u0001/\u0001/\u0001"+
+    "0\u00010\u00010\u00011\u00011\u00011\u00011\u00012\u00012\u00012\u0001"+
+    "3\u00013\u00013\u00014\u00014\u00014\u00014\u00034\u020b\b4\u00014\u0001"+
+    "4\u00014\u00014\u00054\u0211\b4\n4\f4\u0214\t4\u00034\u0216\b4\u00015"+
+    "\u00015\u00015\u00035\u021b\b5\u00015\u00015\u00016\u00016\u00016\u0001"+
+    "6\u00016\u00016\u0000\u0004\u0002\n\u0010\u00127\u0000\u0002\u0004\u0006"+
+    "\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,."+
+    "02468:<>@BDFHJLNPRTVXZ\\^`bdfhjl\u0000\u0007\u0001\u0000?@\u0001\u0000"+
+    "AC\u0001\u0000FG\u0002\u0000##\'\'\u0001\u0000*+\u0002\u0000))77\u0002"+
+    "\u000088:>\u023e\u0000n\u0001\u0000\u0000\u0000\u0002q\u0001\u0000\u0000"+
+    "\u0000\u0004\u0082\u0001\u0000\u0000\u0000\u0006\u0092\u0001\u0000\u0000"+
+    "\u0000\b\u0094\u0001\u0000\u0000\u0000\n\u00b3\u0001\u0000\u0000\u0000"+
+    "\f\u00ce\u0001\u0000\u0000\u0000\u000e\u00d5\u0001\u0000\u0000\u0000\u0010"+
+    "\u00db\u0001\u0000\u0000\u0000\u0012\u00f0\u0001\u0000\u0000\u0000\u0014"+
+    "\u00fa\u0001\u0000\u0000\u0000\u0016\u0109\u0001\u0000\u0000\u0000\u0018"+
+    "\u010b\u0001\u0000\u0000\u0000\u001a\u010e\u0001\u0000\u0000\u0000\u001c"+
+    "\u011b\u0001\u0000\u0000\u0000\u001e\u011d\u0001\u0000\u0000\u0000 \u0129"+
+    "\u0001\u0000\u0000\u0000\"\u012d\u0001\u0000\u0000\u0000$\u012f\u0001"+
+    "\u0000\u0000\u0000&\u0138\u0001\u0000\u0000\u0000(\u013c\u0001\u0000\u0000"+
+    "\u0000*\u014c\u0001\u0000\u0000\u0000,\u014f\u0001\u0000\u0000\u0000."+
+    "\u0157\u0001\u0000\u0000\u00000\u015d\u0001\u0000\u0000\u00002\u0165\u0001"+
+    "\u0000\u0000\u00004\u016d\u0001\u0000\u0000\u00006\u0175\u0001\u0000\u0000"+
+    "\u00008\u0177\u0001\u0000\u0000\u0000:\u01a3\u0001\u0000\u0000\u0000<"+
+    "\u01a5\u0001\u0000\u0000\u0000>\u01a8\u0001\u0000\u0000\u0000@\u01b1\u0001"+
+    "\u0000\u0000\u0000B\u01b9\u0001\u0000\u0000\u0000D\u01bc\u0001\u0000\u0000"+
+    "\u0000F\u01bf\u0001\u0000\u0000\u0000H\u01c8\u0001\u0000\u0000\u0000J"+
+    "\u01cc\u0001\u0000\u0000\u0000L\u01d2\u0001\u0000\u0000\u0000N\u01d6\u0001"+
+    "\u0000\u0000\u0000P\u01d9\u0001\u0000\u0000\u0000R\u01e1\u0001\u0000\u0000"+
+    "\u0000T\u01e5\u0001\u0000\u0000\u0000V\u01e9\u0001\u0000\u0000\u0000X"+
+    "\u01ec\u0001\u0000\u0000\u0000Z\u01f1\u0001\u0000\u0000\u0000\\\u01f5"+
+    "\u0001\u0000\u0000\u0000^\u01f7\u0001\u0000\u0000\u0000`\u01f9\u0001\u0000"+
+    "\u0000\u0000b\u01fc\u0001\u0000\u0000\u0000d\u0200\u0001\u0000\u0000\u0000"+
+    "f\u0203\u0001\u0000\u0000\u0000h\u0206\u0001\u0000\u0000\u0000j\u021a"+
+    "\u0001\u0000\u0000\u0000l\u021e\u0001\u0000\u0000\u0000no\u0003\u0002"+
+    "\u0001\u0000op\u0005\u0000\u0000\u0001p\u0001\u0001\u0000\u0000\u0000"+
+    "qr\u0006\u0001\uffff\uffff\u0000rs\u0003\u0004\u0002\u0000sy\u0001\u0000"+
+    "\u0000\u0000tu\n\u0001\u0000\u0000uv\u0005\u001d\u0000\u0000vx\u0003\u0006"+
+    "\u0003\u0000wt\u0001\u0000\u0000\u0000x{\u0001\u0000\u0000\u0000yw\u0001"+
+    "\u0000\u0000\u0000yz\u0001\u0000\u0000\u0000z\u0003\u0001\u0000\u0000"+
+    "\u0000{y\u0001\u0000\u0000\u0000|\u0083\u0003`0\u0000}\u0083\u0003\u001e"+
+    "\u000f\u0000~\u0083\u0003\u0018\f\u0000\u007f\u0083\u0003(\u0014\u0000"+
+    "\u0080\u0083\u0003d2\u0000\u0081\u0083\u0003f3\u0000\u0082|\u0001\u0000"+
+    "\u0000\u0000\u0082}\u0001\u0000\u0000\u0000\u0082~\u0001\u0000\u0000\u0000"+
+    "\u0082\u007f\u0001\u0000\u0000\u0000\u0082\u0080\u0001\u0000\u0000\u0000"+
+    "\u0082\u0081\u0001\u0000\u0000\u0000\u0083\u0005\u0001\u0000\u0000\u0000"+
+    "\u0084\u0093\u0003*\u0015\u0000\u0085\u0093\u0003.\u0017\u0000\u0086\u0093"+
+    "\u0003<\u001e\u0000\u0087\u0093\u0003l6\u0000\u0088\u0093\u0003B!\u0000"+
+    "\u0089\u0093\u0003>\u001f\u0000\u008a\u0093\u0003,\u0016\u0000\u008b\u0093"+
+    "\u0003\b\u0004\u0000\u008c\u0093\u0003D\"\u0000\u008d\u0093\u0003F#\u0000"+
+    "\u008e\u0093\u0003J%\u0000\u008f\u0093\u0003L&\u0000\u0090\u0093\u0003"+
+    "h4\u0000\u0091\u0093\u0003N\'\u0000\u0092\u0084\u0001\u0000\u0000\u0000"+
+    "\u0092\u0085\u0001\u0000\u0000\u0000\u0092\u0086\u0001\u0000\u0000\u0000"+
+    "\u0092\u0087\u0001\u0000\u0000\u0000\u0092\u0088\u0001\u0000\u0000\u0000"+
+    "\u0092\u0089\u0001\u0000\u0000\u0000\u0092\u008a\u0001\u0000\u0000\u0000"+
+    "\u0092\u008b\u0001\u0000\u0000\u0000\u0092\u008c\u0001\u0000\u0000\u0000"+
+    "\u0092\u008d\u0001\u0000\u0000\u0000\u0092\u008e\u0001\u0000\u0000\u0000"+
+    "\u0092\u008f\u0001\u0000\u0000\u0000\u0092\u0090\u0001\u0000\u0000\u0000"+
+    "\u0092\u0091\u0001\u0000\u0000\u0000\u0093\u0007\u0001\u0000\u0000\u0000"+
+    "\u0094\u0095\u0005\u0014\u0000\u0000\u0095\u0096\u0003\n\u0005\u0000\u0096"+
+    "\t\u0001\u0000\u0000\u0000\u0097\u0098\u0006\u0005\uffff\uffff\u0000\u0098"+
+    "\u0099\u00050\u0000\u0000\u0099\u00b4\u0003\n\u0005\u0007\u009a\u00b4"+
+    "\u0003\u000e\u0007\u0000\u009b\u00b4\u0003\f\u0006\u0000\u009c\u009e\u0003"+
+    "\u000e\u0007\u0000\u009d\u009f\u00050\u0000\u0000\u009e\u009d\u0001\u0000"+
+    "\u0000\u0000\u009e\u009f\u0001\u0000\u0000\u0000\u009f\u00a0\u0001\u0000"+
+    "\u0000\u0000\u00a0\u00a1\u0005-\u0000\u0000\u00a1\u00a2\u0005,\u0000\u0000"+
+    "\u00a2\u00a7\u0003\u000e\u0007\u0000\u00a3\u00a4\u0005&\u0000\u0000\u00a4"+
+    "\u00a6\u0003\u000e\u0007\u0000\u00a5\u00a3\u0001\u0000\u0000\u0000\u00a6"+
+    "\u00a9\u0001\u0000\u0000\u0000\u00a7\u00a5\u0001\u0000\u0000\u0000\u00a7"+
+    "\u00a8\u0001\u0000\u0000\u0000\u00a8\u00aa\u0001\u0000\u0000\u0000\u00a9"+
+    "\u00a7\u0001\u0000\u0000\u0000\u00aa\u00ab\u00056\u0000\u0000\u00ab\u00b4"+
+    "\u0001\u0000\u0000\u0000\u00ac\u00ad\u0003\u000e\u0007\u0000\u00ad\u00af"+
+    "\u0005.\u0000\u0000\u00ae\u00b0\u00050\u0000\u0000\u00af\u00ae\u0001\u0000"+
+    "\u0000\u0000\u00af\u00b0\u0001\u0000\u0000\u0000\u00b0\u00b1\u0001\u0000"+
+    "\u0000\u0000\u00b1\u00b2\u00051\u0000\u0000\u00b2\u00b4\u0001\u0000\u0000"+
+    "\u0000\u00b3\u0097\u0001\u0000\u0000\u0000\u00b3\u009a\u0001\u0000\u0000"+
+    "\u0000\u00b3\u009b\u0001\u0000\u0000\u0000\u00b3\u009c\u0001\u0000\u0000"+
+    "\u0000\u00b3\u00ac\u0001\u0000\u0000\u0000\u00b4\u00bd\u0001\u0000\u0000"+
+    "\u0000\u00b5\u00b6\n\u0004\u0000\u0000\u00b6\u00b7\u0005\"\u0000\u0000"+
+    "\u00b7\u00bc\u0003\n\u0005\u0005\u00b8\u00b9\n\u0003\u0000\u0000\u00b9"+
+    "\u00ba\u00053\u0000\u0000\u00ba\u00bc\u0003\n\u0005\u0004\u00bb\u00b5"+
+    "\u0001\u0000\u0000\u0000\u00bb\u00b8\u0001\u0000\u0000\u0000\u00bc\u00bf"+
+    "\u0001\u0000\u0000\u0000\u00bd\u00bb\u0001\u0000\u0000\u0000\u00bd\u00be"+
+    "\u0001\u0000\u0000\u0000\u00be\u000b\u0001\u0000\u0000\u0000\u00bf\u00bd"+
+    "\u0001\u0000\u0000\u0000\u00c0\u00c2\u0003\u000e\u0007\u0000\u00c1\u00c3"+
+    "\u00050\u0000\u0000\u00c2\u00c1\u0001\u0000\u0000\u0000\u00c2\u00c3\u0001"+
+    "\u0000\u0000\u0000\u00c3\u00c4\u0001\u0000\u0000\u0000\u00c4\u00c5\u0005"+
+    "/\u0000\u0000\u00c5\u00c6\u0003\\.\u0000\u00c6\u00cf\u0001\u0000\u0000"+
+    "\u0000\u00c7\u00c9\u0003\u000e\u0007\u0000\u00c8\u00ca\u00050\u0000\u0000"+
+    "\u00c9\u00c8\u0001\u0000\u0000\u0000\u00c9\u00ca\u0001\u0000\u0000\u0000"+
+    "\u00ca\u00cb\u0001\u0000\u0000\u0000\u00cb\u00cc\u00055\u0000\u0000\u00cc"+
+    "\u00cd\u0003\\.\u0000\u00cd\u00cf\u0001\u0000\u0000\u0000\u00ce\u00c0"+
+    "\u0001\u0000\u0000\u0000\u00ce\u00c7\u0001\u0000\u0000\u0000\u00cf\r\u0001"+
+    "\u0000\u0000\u0000\u00d0\u00d6\u0003\u0010\b\u0000\u00d1\u00d2\u0003\u0010"+
+    "\b\u0000\u00d2\u00d3\u0003^/\u0000\u00d3\u00d4\u0003\u0010\b\u0000\u00d4"+
+    "\u00d6\u0001\u0000\u0000\u0000\u00d5\u00d0\u0001\u0000\u0000\u0000\u00d5"+
+    "\u00d1\u0001\u0000\u0000\u0000\u00d6\u000f\u0001\u0000\u0000\u0000\u00d7"+
+    "\u00d8\u0006\b\uffff\uffff\u0000\u00d8\u00dc\u0003\u0012\t\u0000\u00d9"+
+    "\u00da\u0007\u0000\u0000\u0000\u00da\u00dc\u0003\u0010\b\u0003\u00db\u00d7"+
+    "\u0001\u0000\u0000\u0000\u00db\u00d9\u0001\u0000\u0000\u0000\u00dc\u00e5"+
+    "\u0001\u0000\u0000\u0000\u00dd\u00de\n\u0002\u0000\u0000\u00de\u00df\u0007"+
+    "\u0001\u0000\u0000\u00df\u00e4\u0003\u0010\b\u0003\u00e0\u00e1\n\u0001"+
+    "\u0000\u0000\u00e1\u00e2\u0007\u0000\u0000\u0000\u00e2\u00e4\u0003\u0010"+
+    "\b\u0002\u00e3\u00dd\u0001\u0000\u0000\u0000\u00e3\u00e0\u0001\u0000\u0000"+
+    "\u0000\u00e4\u00e7\u0001\u0000\u0000\u0000\u00e5\u00e3\u0001\u0000\u0000"+
+    "\u0000\u00e5\u00e6\u0001\u0000\u0000\u0000\u00e6\u0011\u0001\u0000\u0000"+
+    "\u0000\u00e7\u00e5\u0001\u0000\u0000\u0000\u00e8\u00e9\u0006\t\uffff\uffff"+
+    "\u0000\u00e9\u00f1\u0003:\u001d\u0000\u00ea\u00f1\u00030\u0018\u0000\u00eb"+
+    "\u00f1\u0003\u0014\n\u0000\u00ec\u00ed\u0005,\u0000\u0000\u00ed\u00ee"+
+    "\u0003\n\u0005\u0000\u00ee\u00ef\u00056\u0000\u0000\u00ef\u00f1\u0001"+
+    "\u0000\u0000\u0000\u00f0\u00e8\u0001\u0000\u0000\u0000\u00f0\u00ea\u0001"+
+    "\u0000\u0000\u0000\u00f0\u00eb\u0001\u0000\u0000\u0000\u00f0\u00ec\u0001"+
+    "\u0000\u0000\u0000\u00f1\u00f7\u0001\u0000\u0000\u0000\u00f2\u00f3\n\u0001"+
+    "\u0000\u0000\u00f3\u00f4\u0005%\u0000\u0000\u00f4\u00f6\u0003\u0016\u000b"+
+    "\u0000\u00f5\u00f2\u0001\u0000\u0000\u0000\u00f6\u00f9\u0001\u0000\u0000"+
+    "\u0000\u00f7\u00f5\u0001\u0000\u0000\u0000\u00f7\u00f8\u0001\u0000\u0000"+
+    "\u0000\u00f8\u0013\u0001\u0000\u0000\u0000\u00f9\u00f7\u0001\u0000\u0000"+
+    "\u0000\u00fa\u00fb\u00036\u001b\u0000\u00fb\u0105\u0005,\u0000\u0000\u00fc"+
+    "\u0106\u0005A\u0000\u0000\u00fd\u0102\u0003\n\u0005\u0000\u00fe\u00ff"+
+    "\u0005&\u0000\u0000\u00ff\u0101\u0003\n\u0005\u0000\u0100\u00fe\u0001"+
+    "\u0000\u0000\u0000\u0101\u0104\u0001\u0000\u0000\u0000\u0102\u0100\u0001"+
+    "\u0000\u0000\u0000\u0102\u0103\u0001\u0000\u0000\u0000\u0103\u0106\u0001"+
+    "\u0000\u0000\u0000\u0104\u0102\u0001\u0000\u0000\u0000\u0105\u00fc\u0001"+
+    "\u0000\u0000\u0000\u0105\u00fd\u0001\u0000\u0000\u0000\u0105\u0106\u0001"+
+    "\u0000\u0000\u0000\u0106\u0107\u0001\u0000\u0000\u0000\u0107\u0108\u0005"+
+    "6\u0000\u0000\u0108\u0015\u0001\u0000\u0000\u0000\u0109\u010a\u00036\u001b"+
+    "\u0000\u010a\u0017\u0001\u0000\u0000\u0000\u010b\u010c\u0005\u0010\u0000"+
+    "\u0000\u010c\u010d\u0003\u001a\r\u0000\u010d\u0019\u0001\u0000\u0000\u0000"+
+    "\u010e\u0113\u0003\u001c\u000e\u0000\u010f\u0110\u0005&\u0000\u0000\u0110"+
+    "\u0112\u0003\u001c\u000e\u0000\u0111\u010f\u0001\u0000\u0000\u0000\u0112"+
+    "\u0115\u0001\u0000\u0000\u0000\u0113\u0111\u0001\u0000\u0000\u0000\u0113"+
+    "\u0114\u0001\u0000\u0000\u0000\u0114\u001b\u0001\u0000\u0000\u0000\u0115"+
+    "\u0113\u0001\u0000\u0000\u0000\u0116\u011c\u0003\n\u0005\u0000\u0117\u0118"+
+    "\u00030\u0018\u0000\u0118\u0119\u0005$\u0000\u0000\u0119\u011a\u0003\n"+
+    "\u0005\u0000\u011a\u011c\u0001\u0000\u0000\u0000\u011b\u0116\u0001\u0000"+
+    "\u0000\u0000\u011b\u0117\u0001\u0000\u0000\u0000\u011c\u001d\u0001\u0000"+
+    "\u0000\u0000\u011d\u011e\u0005\u0006\u0000\u0000\u011e\u0123\u0003 \u0010"+
+    "\u0000\u011f\u0120\u0005&\u0000\u0000\u0120\u0122\u0003 \u0010\u0000\u0121"+
+    "\u011f\u0001\u0000\u0000\u0000\u0122\u0125\u0001\u0000\u0000\u0000\u0123"+
+    "\u0121\u0001\u0000\u0000\u0000\u0123\u0124\u0001\u0000\u0000\u0000\u0124"+
+    "\u0127\u0001\u0000\u0000\u0000\u0125\u0123\u0001\u0000\u0000\u0000\u0126"+
+    "\u0128\u0003\"\u0011\u0000\u0127\u0126\u0001\u0000\u0000\u0000\u0127\u0128"+
+    "\u0001\u0000\u0000\u0000\u0128\u001f\u0001\u0000\u0000\u0000\u0129\u012a"+
+    "\u0005\u0019\u0000\u0000\u012a!\u0001\u0000\u0000\u0000\u012b\u012e\u0003"+
+    "$\u0012\u0000\u012c\u012e\u0003&\u0013\u0000\u012d\u012b\u0001\u0000\u0000"+
+    "\u0000\u012d\u012c\u0001\u0000\u0000\u0000\u012e#\u0001\u0000\u0000\u0000"+
+    "\u012f\u0130\u0005K\u0000\u0000\u0130\u0135\u0003 \u0010\u0000\u0131\u0132"+
+    "\u0005&\u0000\u0000\u0132\u0134\u0003 \u0010\u0000\u0133\u0131\u0001\u0000"+
+    "\u0000\u0000\u0134\u0137\u0001\u0000\u0000\u0000\u0135\u0133\u0001\u0000"+
+    "\u0000\u0000\u0135\u0136\u0001\u0000\u0000\u0000\u0136%\u0001\u0000\u0000"+
+    "\u0000\u0137\u0135\u0001\u0000\u0000\u0000\u0138\u0139\u0005D\u0000\u0000"+
+    "\u0139\u013a\u0003$\u0012\u0000\u013a\u013b\u0005E\u0000\u0000\u013b\'"+
+    "\u0001\u0000\u0000\u0000\u013c\u013d\u0005\r\u0000\u0000\u013d\u0142\u0003"+
+    " \u0010\u0000\u013e\u013f\u0005&\u0000\u0000\u013f\u0141\u0003 \u0010"+
+    "\u0000\u0140\u013e\u0001\u0000\u0000\u0000\u0141\u0144\u0001\u0000\u0000"+
+    "\u0000\u0142\u0140\u0001\u0000\u0000\u0000\u0142\u0143\u0001\u0000\u0000"+
+    "\u0000\u0143\u0146\u0001\u0000\u0000\u0000\u0144\u0142\u0001\u0000\u0000"+
+    "\u0000\u0145\u0147\u0003\u001a\r\u0000\u0146\u0145\u0001\u0000\u0000\u0000"+
+    "\u0146\u0147\u0001\u0000\u0000\u0000\u0147\u014a\u0001\u0000\u0000\u0000"+
+    "\u0148\u0149\u0005!\u0000\u0000\u0149\u014b\u0003\u001a\r\u0000\u014a"+
+    "\u0148\u0001\u0000\u0000\u0000\u014a\u014b\u0001\u0000\u0000\u0000\u014b"+
+    ")\u0001\u0000\u0000\u0000\u014c\u014d\u0005\u0004\u0000\u0000\u014d\u014e"+
+    "\u0003\u001a\r\u0000\u014e+\u0001\u0000\u0000\u0000\u014f\u0151\u0005"+
+    "\u0013\u0000\u0000\u0150\u0152\u0003\u001a\r\u0000\u0151\u0150\u0001\u0000"+
+    "\u0000\u0000\u0151\u0152\u0001\u0000\u0000\u0000\u0152\u0155\u0001\u0000"+
+    "\u0000\u0000\u0153\u0154\u0005!\u0000\u0000\u0154\u0156\u0003\u001a\r"+
+    "\u0000\u0155\u0153\u0001\u0000\u0000\u0000\u0155\u0156\u0001\u0000\u0000"+
+    "\u0000\u0156-\u0001\u0000\u0000\u0000\u0157\u0158\u0005\b\u0000\u0000"+
+    "\u0158\u015b\u0003\u001a\r\u0000\u0159\u015a\u0005!\u0000\u0000\u015a"+
+    "\u015c\u0003\u001a\r\u0000\u015b\u0159\u0001\u0000\u0000\u0000\u015b\u015c"+
+    "\u0001\u0000\u0000\u0000\u015c/\u0001\u0000\u0000\u0000\u015d\u0162\u0003"+
+    "6\u001b\u0000\u015e\u015f\u0005(\u0000\u0000\u015f\u0161\u00036\u001b"+
+    "\u0000\u0160\u015e\u0001\u0000\u0000\u0000\u0161\u0164\u0001\u0000\u0000"+
+    "\u0000\u0162\u0160\u0001\u0000\u0000\u0000\u0162\u0163\u0001\u0000\u0000"+
+    "\u0000\u01631\u0001\u0000\u0000\u0000\u0164\u0162\u0001\u0000\u0000\u0000"+
+    "\u0165\u016a\u00038\u001c\u0000\u0166\u0167\u0005(\u0000\u0000\u0167\u0169"+
+    "\u00038\u001c\u0000\u0168\u0166\u0001\u0000\u0000\u0000\u0169\u016c\u0001"+
+    "\u0000\u0000\u0000\u016a\u0168\u0001\u0000\u0000\u0000\u016a\u016b\u0001"+
+    "\u0000\u0000\u0000\u016b3\u0001\u0000\u0000\u0000\u016c\u016a\u0001\u0000"+
+    "\u0000\u0000\u016d\u0172\u00032\u0019\u0000\u016e\u016f\u0005&\u0000\u0000"+
+    "\u016f\u0171\u00032\u0019\u0000\u0170\u016e\u0001\u0000\u0000\u0000\u0171"+
+    "\u0174\u0001\u0000\u0000\u0000\u0172\u0170\u0001\u0000\u0000\u0000\u0172"+
+    "\u0173\u0001\u0000\u0000\u0000\u01735\u0001\u0000\u0000\u0000\u0174\u0172"+
+    "\u0001\u0000\u0000\u0000\u0175\u0176\u0007\u0002\u0000\u0000\u01767\u0001"+
+    "\u0000\u0000\u0000\u0177\u0178\u0005O\u0000\u0000\u01789\u0001\u0000\u0000"+
+    "\u0000\u0179\u01a4\u00051\u0000\u0000\u017a\u017b\u0003Z-\u0000\u017b"+
+    "\u017c\u0005F\u0000\u0000\u017c\u01a4\u0001\u0000\u0000\u0000\u017d\u01a4"+
+    "\u0003X,\u0000\u017e\u01a4\u0003Z-\u0000\u017f\u01a4\u0003T*\u0000\u0180"+
+    "\u01a4\u00054\u0000\u0000\u0181\u01a4\u0003\\.\u0000\u0182\u0183\u0005"+
+    "D\u0000\u0000\u0183\u0188\u0003V+\u0000\u0184\u0185\u0005&\u0000\u0000"+
+    "\u0185\u0187\u0003V+\u0000\u0186\u0184\u0001\u0000\u0000\u0000\u0187\u018a"+
+    "\u0001\u0000\u0000\u0000\u0188\u0186\u0001\u0000\u0000\u0000\u0188\u0189"+
+    "\u0001\u0000\u0000\u0000\u0189\u018b\u0001\u0000\u0000\u0000\u018a\u0188"+
+    "\u0001\u0000\u0000\u0000\u018b\u018c\u0005E\u0000\u0000\u018c\u01a4\u0001"+
+    "\u0000\u0000\u0000\u018d\u018e\u0005D\u0000\u0000\u018e\u0193\u0003T*"+
+    "\u0000\u018f\u0190\u0005&\u0000\u0000\u0190\u0192\u0003T*\u0000\u0191"+
+    "\u018f\u0001\u0000\u0000\u0000\u0192\u0195\u0001\u0000\u0000\u0000\u0193"+
+    "\u0191\u0001\u0000\u0000\u0000\u0193\u0194\u0001\u0000\u0000\u0000\u0194"+
+    "\u0196\u0001\u0000\u0000\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196"+
+    "\u0197\u0005E\u0000\u0000\u0197\u01a4\u0001\u0000\u0000\u0000\u0198\u0199"+
+    "\u0005D\u0000\u0000\u0199\u019e\u0003\\.\u0000\u019a\u019b\u0005&\u0000"+
+    "\u0000\u019b\u019d\u0003\\.\u0000\u019c\u019a\u0001\u0000\u0000\u0000"+
+    "\u019d\u01a0\u0001\u0000\u0000\u0000\u019e\u019c\u0001\u0000\u0000\u0000"+
+    "\u019e\u019f\u0001\u0000\u0000\u0000\u019f\u01a1\u0001\u0000\u0000\u0000"+
+    "\u01a0\u019e\u0001\u0000\u0000\u0000\u01a1\u01a2\u0005E\u0000\u0000\u01a2"+
+    "\u01a4\u0001\u0000\u0000\u0000\u01a3\u0179\u0001\u0000\u0000\u0000\u01a3"+
+    "\u017a\u0001\u0000\u0000\u0000\u01a3\u017d\u0001\u0000\u0000\u0000\u01a3"+
+    "\u017e\u0001\u0000\u0000\u0000\u01a3\u017f\u0001\u0000\u0000\u0000\u01a3"+
+    "\u0180\u0001\u0000\u0000\u0000\u01a3\u0181\u0001\u0000\u0000\u0000\u01a3"+
+    "\u0182\u0001\u0000\u0000\u0000\u01a3\u018d\u0001\u0000\u0000\u0000\u01a3"+
+    "\u0198\u0001\u0000\u0000\u0000\u01a4;\u0001\u0000\u0000\u0000\u01a5\u01a6"+
+    "\u0005\n\u0000\u0000\u01a6\u01a7\u0005\u001f\u0000\u0000\u01a7=\u0001"+
+    "\u0000\u0000\u0000\u01a8\u01a9\u0005\u0012\u0000\u0000\u01a9\u01ae\u0003"+
+    "@ \u0000\u01aa\u01ab\u0005&\u0000\u0000\u01ab\u01ad\u0003@ \u0000\u01ac"+
+    "\u01aa\u0001\u0000\u0000\u0000\u01ad\u01b0\u0001\u0000\u0000\u0000\u01ae"+
+    "\u01ac\u0001\u0000\u0000\u0000\u01ae\u01af\u0001\u0000\u0000\u0000\u01af"+
+    "?\u0001\u0000\u0000\u0000\u01b0\u01ae\u0001\u0000\u0000\u0000\u01b1\u01b3"+
+    "\u0003\n\u0005\u0000\u01b2\u01b4\u0007\u0003\u0000\u0000\u01b3\u01b2\u0001"+
+    "\u0000\u0000\u0000\u01b3\u01b4\u0001\u0000\u0000\u0000\u01b4\u01b7\u0001"+
+    "\u0000\u0000\u0000\u01b5\u01b6\u00052\u0000\u0000\u01b6\u01b8\u0007\u0004"+
+    "\u0000\u0000\u01b7\u01b5\u0001\u0000\u0000\u0000\u01b7\u01b8\u0001\u0000"+
+    "\u0000\u0000\u01b8A\u0001\u0000\u0000\u0000\u01b9\u01ba\u0005\t\u0000"+
+    "\u0000\u01ba\u01bb\u00034\u001a\u0000\u01bbC\u0001\u0000\u0000\u0000\u01bc"+
+    "\u01bd\u0005\u0002\u0000\u0000\u01bd\u01be\u00034\u001a\u0000\u01beE\u0001"+
+    "\u0000\u0000\u0000\u01bf\u01c0\u0005\u000f\u0000\u0000\u01c0\u01c5\u0003"+
+    "H$\u0000\u01c1\u01c2\u0005&\u0000\u0000\u01c2\u01c4\u0003H$\u0000\u01c3"+
+    "\u01c1\u0001\u0000\u0000\u0000\u01c4\u01c7\u0001\u0000\u0000\u0000\u01c5"+
+    "\u01c3\u0001\u0000\u0000\u0000\u01c5\u01c6\u0001\u0000\u0000\u0000\u01c6"+
+    "G\u0001\u0000\u0000\u0000\u01c7\u01c5\u0001\u0000\u0000\u0000\u01c8\u01c9"+
+    "\u00032\u0019\u0000\u01c9\u01ca\u0005S\u0000\u0000\u01ca\u01cb\u00032"+
+    "\u0019\u0000\u01cbI\u0001\u0000\u0000\u0000\u01cc\u01cd\u0005\u0001\u0000"+
+    "\u0000\u01cd\u01ce\u0003\u0012\t\u0000\u01ce\u01d0\u0003\\.\u0000\u01cf"+
+    "\u01d1\u0003P(\u0000\u01d0\u01cf\u0001\u0000\u0000\u0000\u01d0\u01d1\u0001"+
+    "\u0000\u0000\u0000\u01d1K\u0001\u0000\u0000\u0000\u01d2\u01d3\u0005\u0007"+
+    "\u0000\u0000\u01d3\u01d4\u0003\u0012\t\u0000\u01d4\u01d5\u0003\\.\u0000"+
+    "\u01d5M\u0001\u0000\u0000\u0000\u01d6\u01d7\u0005\u000e\u0000\u0000\u01d7"+
+    "\u01d8\u00030\u0018\u0000\u01d8O\u0001\u0000\u0000\u0000\u01d9\u01de\u0003"+
+    "R)\u0000\u01da\u01db\u0005&\u0000\u0000\u01db\u01dd\u0003R)\u0000\u01dc"+
+    "\u01da\u0001\u0000\u0000\u0000\u01dd\u01e0\u0001\u0000\u0000\u0000\u01de"+
+    "\u01dc\u0001\u0000\u0000\u0000\u01de\u01df\u0001\u0000\u0000\u0000\u01df"+
+    "Q\u0001\u0000\u0000\u0000\u01e0\u01de\u0001\u0000\u0000\u0000\u01e1\u01e2"+
+    "\u00036\u001b\u0000\u01e2\u01e3\u0005$\u0000\u0000\u01e3\u01e4\u0003:"+
+    "\u001d\u0000\u01e4S\u0001\u0000\u0000\u0000\u01e5\u01e6\u0007\u0005\u0000"+
+    "\u0000\u01e6U\u0001\u0000\u0000\u0000\u01e7\u01ea\u0003X,\u0000\u01e8"+
+    "\u01ea\u0003Z-\u0000\u01e9\u01e7\u0001\u0000\u0000\u0000\u01e9\u01e8\u0001"+
+    "\u0000\u0000\u0000\u01eaW\u0001\u0000\u0000\u0000\u01eb\u01ed\u0007\u0000"+
+    "\u0000\u0000\u01ec\u01eb\u0001\u0000\u0000\u0000\u01ec\u01ed\u0001\u0000"+
+    "\u0000\u0000\u01ed\u01ee\u0001\u0000\u0000\u0000\u01ee\u01ef\u0005 \u0000"+
+    "\u0000\u01efY\u0001\u0000\u0000\u0000\u01f0\u01f2\u0007\u0000\u0000\u0000"+
+    "\u01f1\u01f0\u0001\u0000\u0000\u0000\u01f1\u01f2\u0001\u0000\u0000\u0000"+
+    "\u01f2\u01f3\u0001\u0000\u0000\u0000\u01f3\u01f4\u0005\u001f\u0000\u0000"+
+    "\u01f4[\u0001\u0000\u0000\u0000\u01f5\u01f6\u0005\u001e\u0000\u0000\u01f6"+
+    "]\u0001\u0000\u0000\u0000\u01f7\u01f8\u0007\u0006\u0000\u0000\u01f8_\u0001"+
+    "\u0000\u0000\u0000\u01f9\u01fa\u0005\u0005\u0000\u0000\u01fa\u01fb\u0003"+
+    "b1\u0000\u01fba\u0001\u0000\u0000\u0000\u01fc\u01fd\u0005D\u0000\u0000"+
+    "\u01fd\u01fe\u0003\u0002\u0001\u0000\u01fe\u01ff\u0005E\u0000\u0000\u01ff"+
+    "c\u0001\u0000\u0000\u0000\u0200\u0201\u0005\u0011\u0000\u0000\u0201\u0202"+
+    "\u0005i\u0000\u0000\u0202e\u0001\u0000\u0000\u0000\u0203\u0204\u0005\f"+
+    "\u0000\u0000\u0204\u0205\u0005m\u0000\u0000\u0205g\u0001\u0000\u0000\u0000"+
+    "\u0206\u0207\u0005\u0003\u0000\u0000\u0207\u020a\u0005Y\u0000\u0000\u0208"+
+    "\u0209\u0005W\u0000\u0000\u0209\u020b\u00032\u0019\u0000\u020a\u0208\u0001"+
+    "\u0000\u0000\u0000\u020a\u020b\u0001\u0000\u0000\u0000\u020b\u0215\u0001"+
+    "\u0000\u0000\u0000\u020c\u020d\u0005X\u0000\u0000\u020d\u0212\u0003j5"+
+    "\u0000\u020e\u020f\u0005&\u0000\u0000\u020f\u0211\u0003j5\u0000\u0210"+
+    "\u020e\u0001\u0000\u0000\u0000\u0211\u0214\u0001\u0000\u0000\u0000\u0212"+
+    "\u0210\u0001\u0000\u0000\u0000\u0212\u0213\u0001\u0000\u0000\u0000\u0213"+
+    "\u0216\u0001\u0000\u0000\u0000\u0214\u0212\u0001\u0000\u0000\u0000\u0215"+
+    "\u020c\u0001\u0000\u0000\u0000\u0215\u0216\u0001\u0000\u0000\u0000\u0216"+
+    "i\u0001\u0000\u0000\u0000\u0217\u0218\u00032\u0019\u0000\u0218\u0219\u0005"+
+    "$\u0000\u0000\u0219\u021b\u0001\u0000\u0000\u0000\u021a\u0217\u0001\u0000"+
+    "\u0000\u0000\u021a\u021b\u0001\u0000\u0000\u0000\u021b\u021c\u0001\u0000"+
+    "\u0000\u0000\u021c\u021d\u00032\u0019\u0000\u021dk\u0001\u0000\u0000\u0000"+
+    "\u021e\u021f\u0005\u000b\u0000\u0000\u021f\u0220\u0005\u0019\u0000\u0000"+
+    "\u0220\u0221\u0005W\u0000\u0000\u0221\u0222\u00034\u001a\u0000\u0222m"+
+    "\u0001\u0000\u0000\u00004y\u0082\u0092\u009e\u00a7\u00af\u00b3\u00bb\u00bd"+
+    "\u00c2\u00c9\u00ce\u00d5\u00db\u00e3\u00e5\u00f0\u00f7\u0102\u0105\u0113"+
+    "\u011b\u0123\u0127\u012d\u0135\u0142\u0146\u014a\u0151\u0155\u015b\u0162"+
+    "\u016a\u0172\u0188\u0193\u019e\u01a3\u01ae\u01b3\u01b7\u01c5\u01d0\u01de"+
+    "\u01e9\u01ec\u01f1\u020a\u0212\u0215\u021a";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
index e77e0294bba0b..760e8f4cb9cd3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java
@@ -480,6 +480,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterQualifiedNamePatterns(EsqlBaseParser.QualifiedNamePatternsContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitQualifiedNamePatterns(EsqlBaseParser.QualifiedNamePatternsContext ctx) { }
   /**
    * {@inheritDoc}
    *
@@ -912,6 +924,18 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener {
    * <p>The default implementation does nothing.</p>
    */
   @Override public void exitEnrichWithClause(EsqlBaseParser.EnrichWithClauseContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void enterLookupCommand(EsqlBaseParser.LookupCommandContext ctx) { }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation does nothing.</p>
+   */
+  @Override public void exitLookupCommand(EsqlBaseParser.LookupCommandContext ctx) { }
 
   /**
    * {@inheritDoc}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
index 66308d91dce19..aee998364f947 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java
@@ -285,6 +285,13 @@ public class EsqlBaseParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitQualifiedNamePatterns(EsqlBaseParser.QualifiedNamePatternsContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -537,4 +544,11 @@ public class EsqlBaseParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> im
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitEnrichWithClause(EsqlBaseParser.EnrichWithClauseContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitLookupCommand(EsqlBaseParser.LookupCommandContext ctx) { return visitChildren(ctx); }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
index 978ac68670752..0c53191ab9ab7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java
@@ -435,6 +435,16 @@ public interface EsqlBaseParserListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#qualifiedNamePatterns}.
+   * @param ctx the parse tree
+   */
+  void enterQualifiedNamePatterns(EsqlBaseParser.QualifiedNamePatternsContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#qualifiedNamePatterns}.
+   * @param ctx the parse tree
+   */
+  void exitQualifiedNamePatterns(EsqlBaseParser.QualifiedNamePatternsContext ctx);
   /**
    * Enter a parse tree produced by {@link EsqlBaseParser#identifier}.
    * @param ctx the parse tree
@@ -819,4 +829,14 @@ public interface EsqlBaseParserListener extends ParseTreeListener {
    * @param ctx the parse tree
    */
   void exitEnrichWithClause(EsqlBaseParser.EnrichWithClauseContext ctx);
+  /**
+   * Enter a parse tree produced by {@link EsqlBaseParser#lookupCommand}.
+   * @param ctx the parse tree
+   */
+  void enterLookupCommand(EsqlBaseParser.LookupCommandContext ctx);
+  /**
+   * Exit a parse tree produced by {@link EsqlBaseParser#lookupCommand}.
+   * @param ctx the parse tree
+   */
+  void exitLookupCommand(EsqlBaseParser.LookupCommandContext ctx);
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
index bd24afcd28c4a..54f506c9d3b3c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java
@@ -263,6 +263,12 @@ public interface EsqlBaseParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#qualifiedNamePatterns}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitQualifiedNamePatterns(EsqlBaseParser.QualifiedNamePatternsContext ctx);
   /**
    * Visit a parse tree produced by {@link EsqlBaseParser#identifier}.
    * @param ctx the parse tree
@@ -491,4 +497,10 @@ public interface EsqlBaseParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitEnrichWithClause(EsqlBaseParser.EnrichWithClauseContext ctx);
+  /**
+   * Visit a parse tree produced by {@link EsqlBaseParser#lookupCommand}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitLookupCommand(EsqlBaseParser.LookupCommandContext ctx);
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
index c80488cad4190..d026ffbeecbe2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
@@ -71,7 +71,9 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.function.BiFunction;
+import java.util.function.Consumer;
 
+import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.source;
 import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.typedParsing;
@@ -242,6 +244,37 @@ public UnresolvedAttribute visitQualifiedName(EsqlBaseParser.QualifiedNameContex
         return new UnresolvedAttribute(source(ctx), Strings.collectionToDelimitedString(strings, "."));
     }
 
+    @Override
+    public List<NamedExpression> visitQualifiedNamePatterns(EsqlBaseParser.QualifiedNamePatternsContext ctx) {
+        return visitQualifiedNamePatterns(ctx, ne -> {});
+    }
+
+    protected List<NamedExpression> visitQualifiedNamePatterns(
+        EsqlBaseParser.QualifiedNamePatternsContext ctx,
+        Consumer<NamedExpression> checker
+    ) {
+        if (ctx == null) {
+            return emptyList();
+        }
+        List<EsqlBaseParser.QualifiedNamePatternContext> identifiers = ctx.qualifiedNamePattern();
+        List<NamedExpression> names = new ArrayList<>(identifiers.size());
+
+        for (EsqlBaseParser.QualifiedNamePatternContext patternContext : identifiers) {
+            names.add(visitQualifiedNamePattern(patternContext, checker));
+        }
+
+        return names;
+    }
+
+    protected NamedExpression visitQualifiedNamePattern(
+        EsqlBaseParser.QualifiedNamePatternContext patternContext,
+        Consumer<NamedExpression> checker
+    ) {
+        NamedExpression ne = visitQualifiedNamePattern(patternContext);
+        checker.accept(ne);
+        return ne;
+    }
+
     @Override
     public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePatternContext ctx) {
         if (ctx == null) {
@@ -601,11 +634,12 @@ public NamedExpression visitEnrichWithClause(EsqlBaseParser.EnrichWithClauseCont
     }
 
     private NamedExpression enrichFieldName(EsqlBaseParser.QualifiedNamePatternContext ctx) {
-        var name = visitQualifiedNamePattern(ctx);
-        if (name instanceof UnresolvedNamePattern up) {
-            throw new ParsingException(source(ctx), "Using wildcards [*] in ENRICH WITH projections is not allowed [{}]", up.pattern());
-        }
-        return name;
+        return visitQualifiedNamePattern(ctx, ne -> {
+            if (ne instanceof UnresolvedNamePattern up) {
+                var src = ne.source();
+                throw new ParsingException(src, "Using wildcards [*] in ENRICH WITH projections is not allowed [{}]", up.pattern());
+            }
+        });
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
index a7a3d0b816e79..9d13af4f7eba7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
@@ -38,9 +38,9 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern;
 import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.MetadataOptionContext;
-import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.QualifiedNamePatternContext;
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
 import org.elasticsearch.xpack.esql.plan.logical.Drop;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
@@ -317,17 +317,12 @@ public Explain visitExplainCommand(EsqlBaseParser.ExplainCommandContext ctx) {
 
     @Override
     public PlanFactory visitDropCommand(EsqlBaseParser.DropCommandContext ctx) {
-        var identifiers = ctx.qualifiedNamePattern();
-        List<NamedExpression> removals = new ArrayList<>(identifiers.size());
-
-        for (QualifiedNamePatternContext patternContext : identifiers) {
-            NamedExpression ne = visitQualifiedNamePattern(patternContext);
+        List<NamedExpression> removals = visitQualifiedNamePatterns(ctx.qualifiedNamePatterns(), ne -> {
             if (ne instanceof UnresolvedStar) {
                 var src = ne.source();
                 throw new ParsingException(src, "Removing all fields is not allowed [{}]", src.text());
             }
-            removals.add(ne);
-        }
+        });
 
         return child -> new Drop(source(ctx), child, removals);
     }
@@ -340,21 +335,18 @@ public PlanFactory visitRenameCommand(EsqlBaseParser.RenameCommandContext ctx) {
 
     @Override
     public PlanFactory visitKeepCommand(EsqlBaseParser.KeepCommandContext ctx) {
-        var identifiers = ctx.qualifiedNamePattern();
-        List<NamedExpression> projections = new ArrayList<>(identifiers.size());
-        boolean hasSeenStar = false;
-        for (QualifiedNamePatternContext patternContext : identifiers) {
-            NamedExpression ne = visitQualifiedNamePattern(patternContext);
+        final Holder<Boolean> hasSeenStar = new Holder<>(false);
+        List<NamedExpression> projections = visitQualifiedNamePatterns(ctx.qualifiedNamePatterns(), ne -> {
             if (ne instanceof UnresolvedStar) {
-                if (hasSeenStar) {
+                if (hasSeenStar.get()) {
                     var src = ne.source();
                     throw new ParsingException(src, "Cannot specify [*] more than once", src.text());
                 } else {
-                    hasSeenStar = true;
+                    hasSeenStar.set(Boolean.TRUE);
                 }
             }
-            projections.add(ne);
-        }
+        });
+
         return child -> new Keep(source(ctx), child, projections);
     }
 
@@ -437,5 +429,24 @@ public LogicalPlan visitMetricsCommand(EsqlBaseParser.MetricsCommandContext ctx)
         return new EsqlAggregate(source, unresolvedRelation, stats.groupings, stats.aggregates);
     }
 
+    @Override
+    public PlanFactory visitLookupCommand(EsqlBaseParser.LookupCommandContext ctx) {
+        if (false == Build.current().isSnapshot()) {
+            throw new ParsingException(source(ctx), "LOOKUP is in preview and only available in SNAPSHOT build");
+        }
+        var source = source(ctx);
+
+        List<NamedExpression> matchFields = visitQualifiedNamePatterns(ctx.qualifiedNamePatterns(), ne -> {
+            if (ne instanceof UnresolvedNamePattern || ne instanceof UnresolvedStar) {
+                var src = ne.source();
+                throw new ParsingException(src, "Using wildcards [*] in LOOKUP ON is not allowed yet [{}]", src.text());
+            }
+        });
+
+        Literal tableName = new Literal(source, ctx.tableName.getText(), DataTypes.KEYWORD);
+
+        throw new ParsingException(source, "LOOKUP not yet supported");
+    }
+
     interface PlanFactory extends Function<LogicalPlan, LogicalPlan> {}
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index 49a6dfc37cfc1..11ee262c6a682 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -1878,6 +1878,15 @@ public void testInOnText() {
             """, "mapping-multi-field-variation.json", "text");
     }
 
+    public void testLookup() {
+        var e = expectThrows(ParsingException.class, () -> analyze("""
+              FROM test
+            | RENAME languages AS int
+            | LOOKUP int_number_names ON int
+            """));
+        assertThat(e.getMessage(), containsString("LOOKUP not yet supported"));
+    }
+
     private void verifyUnsupported(String query, String errorMessage) {
         verifyUnsupported(query, errorMessage, "mapping-multi-field-variation.json");
     }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml
new file mode 100644
index 0000000000000..e8b372e6d7e8e
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml
@@ -0,0 +1,42 @@
+index named lookup:
+  - do:
+      bulk:
+        index: lookup
+        refresh: true
+        body:
+          - { index: { } }
+          - { f: 1 }
+
+  - do:
+      esql.query:
+        body:
+          query: 'FROM lookup | LIMIT 1'
+  - match: { columns.0.name: f }
+  - match: { columns.0.type: long }
+  - length: { values: 1 }
+  - match: { values.0.0: 1 }
+
+---
+lookup command unsupported:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [method, path, parameters, capabilities]
+          capabilities: [lookup]
+      reason: "LOOKUP command required"
+
+  - do:
+      bulk:
+        index: lookup
+        refresh: true
+        body:
+          - { index: { } }
+          - { f: 1 }
+
+  - do:
+      catch: /LOOKUP not yet supported/
+      esql.query:
+        body:
+          query: 'FROM lookup | LOOKUP a ON foo'

From e931596bcef10b1538d93cc0407388f43b3fbe37 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 28 May 2024 19:27:04 +0100
Subject: [PATCH 014/208] Fix trappy timeouts in watcher actions (#109098)

Relates #107984
---
 .../api/watcher.get_settings.json             |  7 ++++-
 .../rest-api-spec/api/watcher.start.json      |  7 ++++-
 .../rest-api-spec/api/watcher.stop.json       |  7 ++++-
 .../api/watcher.update_settings.json          | 11 +++++++-
 .../org/elasticsearch/TransportVersions.java  |  1 +
 .../actions/put/GetWatcherSettingsAction.java | 24 ++++++++++++----
 .../put/UpdateWatcherSettingsAction.java      | 28 ++++++++++++++++---
 .../service/WatcherServiceRequest.java        |  5 ++--
 .../service/WatcherServiceRequestBuilder.java |  5 ++--
 .../AbstractWatcherIntegrationTestCase.java   |  4 +--
 .../elasticsearch/xpack/watcher/Watcher.java  |  4 +--
 .../action/RestGetWatcherSettingsAction.java  |  3 +-
 .../RestUpdateWatcherSettingsAction.java      | 10 ++++++-
 .../rest/action/RestWatchServiceAction.java   | 12 +++-----
 .../TransportGetWatcherSettingsAction.java    |  2 +-
 .../TransportUpdateWatcherSettingsAction.java |  2 +-
 .../bench/WatcherScheduleEngineBenchmark.java |  3 +-
 17 files changed, 101 insertions(+), 34 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.get_settings.json
index 6bd8cf5fc9228..3ae59c9d024a7 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.get_settings.json
@@ -18,6 +18,11 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.start.json b/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.start.json
index a7884a41198ce..ad0682c8d7b19 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.start.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.start.json
@@ -19,6 +19,11 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.stop.json b/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.stop.json
index c3e85287767fd..b1a67119df153 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.stop.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.stop.json
@@ -19,6 +19,11 @@
         }
       ]
     },
-    "params":{}
+    "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
+      }
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.update_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.update_settings.json
index 5e04e16862a66..5a6a8d4a787ad 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.update_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/watcher.update_settings.json
@@ -18,7 +18,16 @@
         }
       ]
     },
-    "params":{},
+    "params":{
+      "timeout":{
+        "type":"time",
+        "description":"Specify timeout for waiting for acknowledgement from all nodes"
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Specify timeout for connection to master"
+      }
+    },
     "body":{
       "description": "An object with the new index settings",
       "required": true
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index 22460775300f3..f10c079c2e82c 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -179,6 +179,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion FAILURE_STORE_TELEMETRY = def(8_670_00_0);
     public static final TransportVersion ADD_METADATA_FLATTENED_TO_ROLES = def(8_671_00_0);
     public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED = def(8_672_00_0);
+    public static final TransportVersion WATCHER_REQUEST_TIMEOUTS = def(8_673_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/GetWatcherSettingsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/GetWatcherSettingsAction.java
index 902c6db07dc89..f1d046f09b0f7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/GetWatcherSettingsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/GetWatcherSettingsAction.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.watcher.transport.actions.put;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
@@ -14,6 +15,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -30,16 +32,28 @@ public GetWatcherSettingsAction() {
 
     public static class Request extends MasterNodeReadRequest<Request> {
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout) {
+            super(masterNodeTimeout);
         }
 
-        public Request(StreamInput in) throws IOException {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public static Request readFrom(StreamInput in) throws IOException {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.WATCHER_REQUEST_TIMEOUTS)) {
+                return new Request(in);
+            } else {
+                return new Request(TimeValue.THIRTY_SECONDS);
+            }
+        }
+
+        private Request(StreamInput in) throws IOException {
+            super(in);
         }
 
         @Override
-        public void writeTo(StreamOutput out) throws IOException {}
+        public void writeTo(StreamOutput out) throws IOException {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.WATCHER_REQUEST_TIMEOUTS)) {
+                super.writeTo(out);
+            }
+        }
 
         @Override
         public ActionRequestValidationException validate() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/UpdateWatcherSettingsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/UpdateWatcherSettingsAction.java
index b6d999ebbf380..5da714021eb3a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/UpdateWatcherSettingsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/UpdateWatcherSettingsAction.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.watcher.transport.actions.put;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ValidateActions;
@@ -16,6 +17,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV9;
 
 import java.io.IOException;
 import java.util.Map;
@@ -38,18 +41,35 @@ public UpdateWatcherSettingsAction() {
     public static class Request extends AcknowledgedRequest<Request> {
         private final Map<String, Object> settings;
 
-        public Request(Map<String, Object> settings) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, Map<String, Object> settings) {
+            super(masterNodeTimeout, ackTimeout);
             this.settings = settings;
         }
 
-        public Request(StreamInput in) throws IOException {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public static Request readFrom(StreamInput in) throws IOException {
+            if (in.getTransportVersion().onOrAfter(TransportVersions.WATCHER_REQUEST_TIMEOUTS)) {
+                return new Request(in);
+            } else {
+                return new Request(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, in);
+            }
+        }
+
+        private Request(StreamInput in) throws IOException {
+            super(in);
+            this.settings = in.readGenericMap();
+        }
+
+        @UpdateForV9 // bwc no longer required
+        private Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, StreamInput in) throws IOException {
+            super(masterNodeTimeout, ackTimeout);
             this.settings = in.readGenericMap();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
+            if (out.getTransportVersion().onOrAfter(TransportVersions.WATCHER_REQUEST_TIMEOUTS)) {
+                super.writeTo(out);
+            }
             out.writeGenericMap(this.settings);
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceRequest.java
index 449179e4f18f6..4ccbcdf41949a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceRequest.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -29,8 +30,8 @@ public WatcherServiceRequest(StreamInput in) throws IOException {
         command = Command.valueOf(in.readString().toUpperCase(Locale.ROOT));
     }
 
-    public WatcherServiceRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public WatcherServiceRequest(TimeValue masterNodeTimeout) {
+        super(masterNodeTimeout);
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceRequestBuilder.java
index 67284d54e3112..07dd80c167d1b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceRequestBuilder.java
@@ -9,14 +9,15 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.internal.ElasticsearchClient;
+import org.elasticsearch.core.TimeValue;
 
 public class WatcherServiceRequestBuilder extends MasterNodeOperationRequestBuilder<
     WatcherServiceRequest,
     AcknowledgedResponse,
     WatcherServiceRequestBuilder> {
 
-    public WatcherServiceRequestBuilder(ElasticsearchClient client) {
-        super(client, WatcherServiceAction.INSTANCE, new WatcherServiceRequest());
+    public WatcherServiceRequestBuilder(TimeValue masterNodeTimeout, ElasticsearchClient client) {
+        super(client, WatcherServiceAction.INSTANCE, new WatcherServiceRequest(masterNodeTimeout));
     }
 
     /**
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
index 41e23b54b0375..3b9ea0bd18d47 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
@@ -540,7 +540,7 @@ protected void startWatcher() throws Exception {
 
             boolean isAllStateStopped = states.stream().allMatch(w -> w == WatcherState.STOPPED);
             if (isAllStateStopped) {
-                assertAcked(new WatcherServiceRequestBuilder(client()).start().get());
+                assertAcked(new WatcherServiceRequestBuilder(TEST_REQUEST_TIMEOUT, client()).start().get());
                 throw new AssertionError("all nodes are stopped, restarting");
             }
 
@@ -582,7 +582,7 @@ protected void stopWatcher() throws Exception {
 
             boolean isAllStateStarted = states.stream().allMatch(w -> w == WatcherState.STARTED);
             if (isAllStateStarted) {
-                assertAcked(new WatcherServiceRequestBuilder(client()).stop().get());
+                assertAcked(new WatcherServiceRequestBuilder(TEST_REQUEST_TIMEOUT, client()).stop().get());
                 throw new AssertionError("all nodes are started, stopping");
             }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
index 010c3611c1f96..f8f910c38c080 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
@@ -834,7 +834,7 @@ public void prepareForIndicesMigration(ClusterService clusterService, Client cli
             .orElse(false);
 
         if (manuallyStopped == false) {
-            WatcherServiceRequest serviceRequest = new WatcherServiceRequest();
+            WatcherServiceRequest serviceRequest = new WatcherServiceRequest(TimeValue.THIRTY_SECONDS /* TODO should this be longer? */);
             serviceRequest.stop();
             originClient.execute(WatcherServiceAction.INSTANCE, serviceRequest, ActionListener.wrap((response) -> {
                 listener.onResponse(Collections.singletonMap("manually_stopped", manuallyStopped));
@@ -855,7 +855,7 @@ public void indicesMigrationComplete(
         Client originClient = new OriginSettingClient(client, WATCHER_ORIGIN);
         boolean manuallyStopped = (boolean) preUpgradeMetadata.getOrDefault("manually_stopped", false);
         if (manuallyStopped == false) {
-            WatcherServiceRequest serviceRequest = new WatcherServiceRequest();
+            WatcherServiceRequest serviceRequest = new WatcherServiceRequest(TimeValue.THIRTY_SECONDS /* TODO should this be longer? */);
             serviceRequest.start();
             originClient.execute(WatcherServiceAction.INSTANCE, serviceRequest, ActionListener.wrap((response) -> {
                 listener.onResponse(response.isAcknowledged());
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatcherSettingsAction.java
index ca3cfbaaedd72..73a933c9c2e46 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatcherSettingsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatcherSettingsAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.GetWatcherSettingsAction;
 
@@ -33,7 +34,7 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        GetWatcherSettingsAction.Request req = new GetWatcherSettingsAction.Request();
+        GetWatcherSettingsAction.Request req = new GetWatcherSettingsAction.Request(RestUtils.getMasterNodeTimeout(request));
         return channel -> client.execute(GetWatcherSettingsAction.INSTANCE, req, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestUpdateWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestUpdateWatcherSettingsAction.java
index 3861573c1e421..26f64a0918141 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestUpdateWatcherSettingsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestUpdateWatcherSettingsAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction;
 
@@ -33,7 +34,14 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        UpdateWatcherSettingsAction.Request req = new UpdateWatcherSettingsAction.Request(request.contentParser().map());
+        final UpdateWatcherSettingsAction.Request req;
+        try (var contentParser = request.contentParser()) {
+            req = new UpdateWatcherSettingsAction.Request(
+                RestUtils.getMasterNodeTimeout(request),
+                RestUtils.getAckTimeout(request),
+                contentParser.map()
+            );
+        }
         return channel -> client.execute(UpdateWatcherSettingsAction.INSTANCE, req, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java
index 7824f9f46c2f6..f6b9ae9c4031c 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceRequest;
@@ -18,7 +19,6 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
-import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 public class RestWatchServiceAction extends BaseRestHandler {
 
@@ -34,11 +34,8 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        return channel -> client.execute(
-            WatcherServiceAction.INSTANCE,
-            new WatcherServiceRequest().start(),
-            new RestToXContentListener<>(channel)
-        );
+        final var req = new WatcherServiceRequest(RestUtils.getMasterNodeTimeout(request)).start();
+        return channel -> client.execute(WatcherServiceAction.INSTANCE, req, new RestToXContentListener<>(channel));
     }
 
     public static class StopRestHandler extends BaseRestHandler {
@@ -55,8 +52,7 @@ public String getName() {
 
         @Override
         public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-            final WatcherServiceRequest request = new WatcherServiceRequest().stop();
-            request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+            final var request = new WatcherServiceRequest(RestUtils.getMasterNodeTimeout(restRequest)).stop();
             return channel -> client.execute(WatcherServiceAction.INSTANCE, request, new RestToXContentListener<>(channel));
         }
     }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java
index 0c52057100860..f0fe010962697 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java
@@ -46,7 +46,7 @@ public TransportGetWatcherSettingsAction(
             clusterService,
             threadPool,
             actionFilters,
-            GetWatcherSettingsAction.Request::new,
+            GetWatcherSettingsAction.Request::readFrom,
             indexNameExpressionResolver,
             GetWatcherSettingsAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java
index 94ef321806033..8064a17d451e3 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportUpdateWatcherSettingsAction.java
@@ -69,7 +69,7 @@ public TransportUpdateWatcherSettingsAction(
             clusterService,
             threadPool,
             actionFilters,
-            UpdateWatcherSettingsAction.Request::new,
+            UpdateWatcherSettingsAction.Request::readFrom,
             indexNameExpressionResolver,
             AcknowledgedResponse::readFrom,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java
index 670da0b8f788d..53ca2fb3b3a35 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.Percentiles;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPoolStats;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.watcher.WatcherState;
@@ -242,7 +243,7 @@ public void run() {
                         Percentiles percentiles = searchResponse.getAggregations().get("percentile_delay");
                         stats.setDelayPercentiles(percentiles);
                         stats.setAvgJvmUsed(jvmUsedHeapSpace);
-                        new WatcherServiceRequestBuilder(client).stop().get();
+                        new WatcherServiceRequestBuilder(ESTestCase.TEST_REQUEST_TIMEOUT, client).stop().get();
                     }
                 );
             }

From 4275c927f3052723b33f80f2a2cb88e775acf7e6 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 28 May 2024 19:49:05 +0100
Subject: [PATCH 015/208] Fix trappy timeouts in SLM modules (#109122)

Relates #107984
---
 .../action/DeleteSnapshotLifecycleAction.java | 11 ++--
 .../ExecuteSnapshotLifecycleAction.java       | 26 ++-------
 .../ExecuteSnapshotRetentionAction.java       | 16 ++----
 .../action/GetSnapshotLifecycleAction.java    | 10 +---
 .../action/PutSnapshotLifecycleAction.java    | 31 +++-------
 .../xpack/core/slm/action/StartSLMAction.java |  5 +-
 .../xpack/core/slm/action/StopSLMAction.java  |  5 +-
 .../ilm/LifecycleOperationSnapshotTests.java  |  6 +-
 .../xpack/slm/SLMFileSettingsIT.java          | 10 +++-
 .../slm/SLMSnapshotBlockingIntegTests.java    | 56 ++++++++++++++-----
 .../SnapshotLifecycleInitialisationTests.java |  2 +
 .../slm/action/ReservedSnapshotAction.java    | 17 +++++-
 .../RestDeleteSnapshotLifecycleAction.java    | 10 ++--
 .../RestExecuteSnapshotLifecycleAction.java   |  9 +--
 .../RestExecuteSnapshotRetentionAction.java   |  4 +-
 .../RestGetSnapshotLifecycleAction.java       | 10 ++--
 .../RestPutSnapshotLifecycleAction.java       | 10 ++--
 .../xpack/slm/action/RestStartSLMAction.java  |  4 +-
 .../xpack/slm/action/RestStopSLMAction.java   |  4 +-
 ...ransportDeleteSnapshotLifecycleAction.java |  9 ---
 ...vedSnapshotLifecycleStateServiceTests.java |  9 ++-
 .../action/TransportStopSLMActionTests.java   |  2 +-
 22 files changed, 133 insertions(+), 133 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/DeleteSnapshotLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/DeleteSnapshotLifecycleAction.java
index 6e083295b0863..fb461a772f465 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/DeleteSnapshotLifecycleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/DeleteSnapshotLifecycleAction.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -26,19 +27,15 @@ protected DeleteSnapshotLifecycleAction() {
 
     public static class Request extends AcknowledgedRequest<Request> {
 
-        private String lifecycleId;
+        private final String lifecycleId;
 
         public Request(StreamInput in) throws IOException {
             super(in);
             lifecycleId = in.readString();
         }
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-        }
-
-        public Request(String lifecycleId) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String lifecycleId) {
+            super(masterNodeTimeout, ackTimeout);
             this.lifecycleId = Objects.requireNonNull(lifecycleId, "id may not be null");
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/ExecuteSnapshotLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/ExecuteSnapshotLifecycleAction.java
index 442ff6b2bfb66..7793c628fb7cc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/ExecuteSnapshotLifecycleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/ExecuteSnapshotLifecycleAction.java
@@ -10,9 +10,9 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -31,12 +31,12 @@ protected ExecuteSnapshotLifecycleAction() {
         super(NAME);
     }
 
-    public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
+    public static class Request extends AcknowledgedRequest<Request> {
 
-        private String lifecycleId;
+        private final String lifecycleId;
 
-        public Request(String lifecycleId) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String lifecycleId) {
+            super(masterNodeTimeout, ackTimeout);
             this.lifecycleId = lifecycleId;
         }
 
@@ -45,10 +45,6 @@ public Request(StreamInput in) throws IOException {
             lifecycleId = in.readString();
         }
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-        }
-
         public String getLifecycleId() {
             return this.lifecycleId;
         }
@@ -59,13 +55,6 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(lifecycleId);
         }
 
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.endObject();
-            return builder;
-        }
-
         @Override
         public int hashCode() {
             return Objects.hash(lifecycleId);
@@ -82,11 +71,6 @@ public boolean equals(Object obj) {
             Request other = (Request) obj;
             return lifecycleId.equals(other.lifecycleId);
         }
-
-        @Override
-        public String toString() {
-            return Strings.toString(this);
-        }
     }
 
     public static class Response extends ActionResponse implements ToXContentObject {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/ExecuteSnapshotRetentionAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/ExecuteSnapshotRetentionAction.java
index e4d698f48d252..b374b510625f9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/ExecuteSnapshotRetentionAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/ExecuteSnapshotRetentionAction.java
@@ -11,8 +11,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -24,23 +23,16 @@ protected ExecuteSnapshotRetentionAction() {
         super(NAME);
     }
 
-    public static class Request extends AcknowledgedRequest<ExecuteSnapshotRetentionAction.Request> implements ToXContentObject {
+    public static class Request extends AcknowledgedRequest<ExecuteSnapshotRetentionAction.Request> {
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+            super(masterNodeTimeout, ackTimeout);
         }
 
         public Request(StreamInput in) throws IOException {
             super(in);
         }
 
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.endObject();
-            return builder;
-        }
-
         @Override
         public int hashCode() {
             return super.hashCode();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleAction.java
index ad62b155da41c..dd330739d3e69 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSnapshotLifecycleAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyItem;
@@ -34,8 +35,8 @@ public static class Request extends AcknowledgedRequest<GetSnapshotLifecycleActi
 
         private final String[] lifecycleIds;
 
-        public Request(String... lifecycleIds) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String... lifecycleIds) {
+            super(masterNodeTimeout, ackTimeout);
             this.lifecycleIds = Objects.requireNonNull(lifecycleIds, "ids may not be null");
         }
 
@@ -44,11 +45,6 @@ public Request(StreamInput in) throws IOException {
             lifecycleIds = in.readStringArray();
         }
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-            this.lifecycleIds = Strings.EMPTY_ARRAY;
-        }
-
         public String[] getLifecycleIds() {
             return this.lifecycleIds;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/PutSnapshotLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/PutSnapshotLifecycleAction.java
index dd66ff1fd5371..b06f109ef6027 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/PutSnapshotLifecycleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/PutSnapshotLifecycleAction.java
@@ -14,8 +14,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
 
@@ -30,13 +29,13 @@ protected PutSnapshotLifecycleAction() {
         super(NAME);
     }
 
-    public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
+    public static class Request extends AcknowledgedRequest<Request> {
 
-        private String lifecycleId;
-        private SnapshotLifecyclePolicy lifecycle;
+        private final String lifecycleId;
+        private final SnapshotLifecyclePolicy lifecycle;
 
-        public Request(String lifecycleId, SnapshotLifecyclePolicy lifecycle) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String lifecycleId, SnapshotLifecyclePolicy lifecycle) {
+            super(masterNodeTimeout, ackTimeout);
             this.lifecycleId = lifecycleId;
             this.lifecycle = lifecycle;
         }
@@ -47,10 +46,6 @@ public Request(StreamInput in) throws IOException {
             lifecycle = new SnapshotLifecyclePolicy(in);
         }
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-        }
-
         public String getLifecycleId() {
             return this.lifecycleId;
         }
@@ -59,8 +54,8 @@ public SnapshotLifecyclePolicy getLifecycle() {
             return this.lifecycle;
         }
 
-        public static Request parseRequest(String lifecycleId, XContentParser parser) {
-            return new Request(lifecycleId, SnapshotLifecyclePolicy.parse(parser, lifecycleId));
+        public static Request parseRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout, String lifecycleId, XContentParser parser) {
+            return new Request(masterNodeTimeout, ackTimeout, lifecycleId, SnapshotLifecyclePolicy.parse(parser, lifecycleId));
         }
 
         @Override
@@ -75,14 +70,6 @@ public ActionRequestValidationException validate() {
             return lifecycle.validate();
         }
 
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field(lifecycleId, lifecycle);
-            builder.endObject();
-            return builder;
-        }
-
         @Override
         public int hashCode() {
             return Objects.hash(lifecycleId, lifecycle);
@@ -102,7 +89,7 @@ public boolean equals(Object obj) {
 
         @Override
         public String toString() {
-            return Strings.toString(this);
+            return Strings.toString((b, p) -> b.field(lifecycleId, lifecycle));
         }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/StartSLMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/StartSLMAction.java
index 666701ac1f885..bacb8f4cd613e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/StartSLMAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/StartSLMAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -28,8 +29,8 @@ public Request(StreamInput in) throws IOException {
             super(in);
         }
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+            super(masterNodeTimeout, ackTimeout);
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/StopSLMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/StopSLMAction.java
index 4aae048b5e5b6..57bd414bed842 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/StopSLMAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/StopSLMAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -28,8 +29,8 @@ public Request(StreamInput in) throws IOException {
             super(in);
         }
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+            super(masterNodeTimeout, ackTimeout);
         }
 
         @Override
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
index 22eef3d8940a3..aa9da48eb8f8f 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
@@ -63,6 +63,8 @@ public void testModeSnapshotRestore() throws Exception {
         client().execute(
             PutSnapshotLifecycleAction.INSTANCE,
             new PutSnapshotLifecycleAction.Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
                 "slm-policy",
                 new SnapshotLifecyclePolicy(
                     "slm-policy",
@@ -91,7 +93,7 @@ public void testModeSnapshotRestore() throws Exception {
         // Take snapshot
         ExecuteSnapshotLifecycleAction.Response resp = client().execute(
             ExecuteSnapshotLifecycleAction.INSTANCE,
-            new ExecuteSnapshotLifecycleAction.Request("slm-policy")
+            new ExecuteSnapshotLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "slm-policy")
         ).get();
         final String snapshotName = resp.getSnapshotName();
         // Wait for the snapshot to be successful
@@ -110,7 +112,7 @@ public void testModeSnapshotRestore() throws Exception {
         });
 
         assertAcked(client().execute(ILMActions.STOP, new StopILMRequest()).get());
-        assertAcked(client().execute(StopSLMAction.INSTANCE, new StopSLMAction.Request()).get());
+        assertAcked(client().execute(StopSLMAction.INSTANCE, new StopSLMAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)).get());
         assertBusy(() -> assertThat(ilmMode(), equalTo(OperationMode.STOPPED)));
         assertBusy(() -> assertThat(slmMode(), equalTo(OperationMode.STOPPED)));
 
diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMFileSettingsIT.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMFileSettingsIT.java
index 725bf412b3198..c68e7174923f8 100644
--- a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMFileSettingsIT.java
+++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMFileSettingsIT.java
@@ -259,7 +259,7 @@ public void testSettingsApplied() throws Exception {
         assertBusy(() -> {
             GetSnapshotLifecycleAction.Response getResp = client().execute(
                 GetSnapshotLifecycleAction.INSTANCE,
-                new GetSnapshotLifecycleAction.Request(policyName)
+                new GetSnapshotLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policyName)
             ).get();
             logger.info("--> checking for snapshot complete...");
 
@@ -357,7 +357,11 @@ public void testErrorSaved() throws Exception {
     }
 
     private String executePolicy(String policyId) {
-        ExecuteSnapshotLifecycleAction.Request executeReq = new ExecuteSnapshotLifecycleAction.Request(policyId);
+        ExecuteSnapshotLifecycleAction.Request executeReq = new ExecuteSnapshotLifecycleAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            policyId
+        );
         ExecuteSnapshotLifecycleAction.Response resp = null;
         try {
             resp = client().execute(ExecuteSnapshotLifecycleAction.INSTANCE, executeReq).get();
@@ -392,7 +396,7 @@ private PutSnapshotLifecycleAction.Request sampleRestRequest(String name) throws
             var parser = JSON.xContent().createParser(XContentParserConfiguration.EMPTY, bis)
         ) {
             var policy = SnapshotLifecyclePolicy.parse(parser, name);
-            return new PutSnapshotLifecycleAction.Request(name, policy);
+            return new PutSnapshotLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, name, policy);
         }
     }
 }
diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java
index d1e6c56ae1517..3787761a2b287 100644
--- a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java
+++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java
@@ -127,7 +127,7 @@ public void testSnapshotInProgress() throws Exception {
         assertBusy(() -> {
             GetSnapshotLifecycleAction.Response getResp = client().execute(
                 GetSnapshotLifecycleAction.INSTANCE,
-                new GetSnapshotLifecycleAction.Request(policyName)
+                new GetSnapshotLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policyName)
             ).get();
             logger.info("--> checking for in progress snapshot...");
 
@@ -215,7 +215,7 @@ public void testRetentionWhileSnapshotInProgress() throws Exception {
                 logger.info("--> at least one data node has hit the block");
                 GetSnapshotLifecycleAction.Response getResp = client().execute(
                     GetSnapshotLifecycleAction.INSTANCE,
-                    new GetSnapshotLifecycleAction.Request(policyId)
+                    new GetSnapshotLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policyId)
                 ).get();
                 logger.info("--> checking for in progress snapshot...");
 
@@ -235,9 +235,10 @@ public void testRetentionWhileSnapshotInProgress() throws Exception {
             // Run retention
             logger.info("--> triggering retention");
             assertTrue(
-                client().execute(ExecuteSnapshotRetentionAction.INSTANCE, new ExecuteSnapshotRetentionAction.Request())
-                    .get()
-                    .isAcknowledged()
+                client().execute(
+                    ExecuteSnapshotRetentionAction.INSTANCE,
+                    new ExecuteSnapshotRetentionAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+                ).get().isAcknowledged()
             );
 
             logger.info("--> unblocking snapshots");
@@ -313,7 +314,10 @@ public void testRetentionWithMultipleRepositories() throws Exception {
             new SnapshotRetentionConfiguration(null, 1, 2)
         );
         logger.info("-->  start snapshot");
-        client().execute(ExecuteSnapshotLifecycleAction.INSTANCE, new ExecuteSnapshotLifecycleAction.Request(policyId)).get();
+        client().execute(
+            ExecuteSnapshotLifecycleAction.INSTANCE,
+            new ExecuteSnapshotLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policyId)
+        ).get();
         // make sure the SLM history data stream is green and won't not be green for long because of delayed allocation when data nodes
         // are stopped
         ensureGreen(SLM_HISTORY_DATA_STREAM);
@@ -360,7 +364,7 @@ private void testUnsuccessfulSnapshotRetention(boolean partialSuccess) throws Ex
                 logger.info("-->  start snapshot");
                 ActionFuture<ExecuteSnapshotLifecycleAction.Response> snapshotFuture = client().execute(
                     ExecuteSnapshotLifecycleAction.INSTANCE,
-                    new ExecuteSnapshotLifecycleAction.Request(policyId)
+                    new ExecuteSnapshotLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policyId)
                 );
 
                 waitForBlock(internalCluster().getMasterName(), REPO);
@@ -395,7 +399,12 @@ private void testUnsuccessfulSnapshotRetention(boolean partialSuccess) throws Ex
         // Run retention - we'll check the results later to make sure it's had time to run.
         {
             logger.info("--> executing SLM retention");
-            assertAcked(client().execute(ExecuteSnapshotRetentionAction.INSTANCE, new ExecuteSnapshotRetentionAction.Request()).get());
+            assertAcked(
+                client().execute(
+                    ExecuteSnapshotRetentionAction.INSTANCE,
+                    new ExecuteSnapshotRetentionAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+                ).get()
+            );
         }
 
         // Take a successful snapshot
@@ -413,7 +422,7 @@ private void testUnsuccessfulSnapshotRetention(boolean partialSuccess) throws Ex
 
             ActionFuture<ExecuteSnapshotLifecycleAction.Response> snapshotResponse = client().execute(
                 ExecuteSnapshotLifecycleAction.INSTANCE,
-                new ExecuteSnapshotLifecycleAction.Request(policyId)
+                new ExecuteSnapshotLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policyId)
             );
             logger.info("--> waiting for snapshot to complete");
             successfulSnapshotName.set(snapshotResponse.get().getSnapshotName());
@@ -446,7 +455,12 @@ private void testUnsuccessfulSnapshotRetention(boolean partialSuccess) throws Ex
         // Run retention again and make sure the failure was deleted
         {
             logger.info("--> executing SLM retention");
-            assertAcked(client().execute(ExecuteSnapshotRetentionAction.INSTANCE, new ExecuteSnapshotRetentionAction.Request()).get());
+            assertAcked(
+                client().execute(
+                    ExecuteSnapshotRetentionAction.INSTANCE,
+                    new ExecuteSnapshotRetentionAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+                ).get()
+            );
             logger.info("--> waiting for {} snapshot [{}] to be deleted", expectedUnsuccessfulState, failedSnapshotName.get());
             assertBusy(() -> {
                 try {
@@ -497,7 +511,7 @@ public void testSLMRetentionAfterRestore() throws Exception {
         assertBusy(() -> {
             GetSnapshotLifecycleAction.Response getResp = client().execute(
                 GetSnapshotLifecycleAction.INSTANCE,
-                new GetSnapshotLifecycleAction.Request(policyName)
+                new GetSnapshotLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policyName)
             ).get();
             logger.info("--> checking for in progress snapshot...");
 
@@ -518,7 +532,12 @@ public void testSLMRetentionAfterRestore() throws Exception {
         assertThat(resp.status(), equalTo(RestStatus.OK));
 
         logger.info("--> executing SLM retention");
-        assertAcked(client().execute(ExecuteSnapshotRetentionAction.INSTANCE, new ExecuteSnapshotRetentionAction.Request()).get());
+        assertAcked(
+            client().execute(
+                ExecuteSnapshotRetentionAction.INSTANCE,
+                new ExecuteSnapshotRetentionAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+            ).get()
+        );
         logger.info("--> waiting for {} snapshot to be deleted", snapshotName);
         assertBusy(() -> {
             try {
@@ -595,7 +614,12 @@ private void createSnapshotPolicy(
             retention
         );
 
-        PutSnapshotLifecycleAction.Request putLifecycle = new PutSnapshotLifecycleAction.Request(policyName, policy);
+        PutSnapshotLifecycleAction.Request putLifecycle = new PutSnapshotLifecycleAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            policyName,
+            policy
+        );
         try {
             client().execute(PutSnapshotLifecycleAction.INSTANCE, putLifecycle).get();
         } catch (Exception e) {
@@ -608,7 +632,11 @@ private void createSnapshotPolicy(
      * Execute the given policy and return the generated snapshot name
      */
     private String executePolicy(String policyId) {
-        ExecuteSnapshotLifecycleAction.Request executeReq = new ExecuteSnapshotLifecycleAction.Request(policyId);
+        ExecuteSnapshotLifecycleAction.Request executeReq = new ExecuteSnapshotLifecycleAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            policyId
+        );
         ExecuteSnapshotLifecycleAction.Response resp = null;
         try {
             resp = client().execute(ExecuteSnapshotLifecycleAction.INSTANCE, executeReq).get();
diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
index 6d0cd2142fe6e..e5e71a38ce6b4 100644
--- a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
+++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
@@ -69,6 +69,8 @@ public void testSLMIsInRunningModeWhenILMIsDisabled() throws Exception {
         client().execute(
             PutSnapshotLifecycleAction.INSTANCE,
             new Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
                 "snapshot-policy",
                 new SnapshotLifecyclePolicy(
                     "test-policy",
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java
index fe385a6389d55..f14edd89b826d 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotAction.java
@@ -7,12 +7,15 @@
 
 package org.elasticsearch.xpack.slm.action;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
+import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction;
 import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction;
 import org.elasticsearch.xpack.slm.SnapshotLifecycleService;
 
@@ -51,7 +54,13 @@ private Collection<PutSnapshotLifecycleAction.Request> prepare(List<SnapshotLife
         List<Exception> exceptions = new ArrayList<>();
 
         for (var policy : policies) {
-            PutSnapshotLifecycleAction.Request request = new PutSnapshotLifecycleAction.Request(policy.getId(), policy);
+            // timeouts don't matter here
+            PutSnapshotLifecycleAction.Request request = new PutSnapshotLifecycleAction.Request(
+                TimeValue.THIRTY_SECONDS,
+                TimeValue.THIRTY_SECONDS,
+                policy.getId(),
+                policy
+            );
             try {
                 validate(request);
                 SnapshotLifecycleService.validateRepositoryExists(request.getLifecycle().getRepository(), state);
@@ -91,7 +100,11 @@ public TransformState transform(Object source, TransformState prevState) throws
         toDelete.removeAll(entities);
 
         for (var policyToDelete : toDelete) {
-            var task = new TransportDeleteSnapshotLifecycleAction.DeleteSnapshotPolicyTask(policyToDelete);
+            // timeouts don't matter here
+            var task = new TransportDeleteSnapshotLifecycleAction.DeleteSnapshotPolicyTask(
+                new DeleteSnapshotLifecycleAction.Request(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, policyToDelete),
+                ActionListener.noop()
+            );
             state = task.execute(state);
         }
 
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestDeleteSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestDeleteSnapshotLifecycleAction.java
index d3eef098f2083..1dd10ddf74768 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestDeleteSnapshotLifecycleAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestDeleteSnapshotLifecycleAction.java
@@ -36,11 +36,11 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        String lifecycleId = request.param("name");
-        DeleteSnapshotLifecycleAction.Request req = new DeleteSnapshotLifecycleAction.Request(lifecycleId);
-        req.ackTimeout(getAckTimeout(request));
-        req.masterNodeTimeout(getMasterNodeTimeout(request));
-
+        final var req = new DeleteSnapshotLifecycleAction.Request(
+            getMasterNodeTimeout(request),
+            getAckTimeout(request),
+            request.param("name")
+        );
         return channel -> client.execute(DeleteSnapshotLifecycleAction.INSTANCE, req, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotLifecycleAction.java
index 4b5b072101c36..8132c7443abe2 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotLifecycleAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotLifecycleAction.java
@@ -37,10 +37,11 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        String snapLifecycleId = request.param("name");
-        ExecuteSnapshotLifecycleAction.Request req = new ExecuteSnapshotLifecycleAction.Request(snapLifecycleId);
-        req.ackTimeout(getAckTimeout(request));
-        req.masterNodeTimeout(getMasterNodeTimeout(request));
+        final var req = new ExecuteSnapshotLifecycleAction.Request(
+            getMasterNodeTimeout(request),
+            getAckTimeout(request),
+            request.param("name")
+        );
         return channel -> client.execute(ExecuteSnapshotLifecycleAction.INSTANCE, req, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotRetentionAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotRetentionAction.java
index e00854879c32b..b133348fd958a 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotRetentionAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotRetentionAction.java
@@ -36,9 +36,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        ExecuteSnapshotRetentionAction.Request req = new ExecuteSnapshotRetentionAction.Request();
-        req.ackTimeout(getAckTimeout(request));
-        req.masterNodeTimeout(getMasterNodeTimeout(request));
+        final var req = new ExecuteSnapshotRetentionAction.Request(getMasterNodeTimeout(request), getAckTimeout(request));
         return channel -> client.execute(ExecuteSnapshotRetentionAction.INSTANCE, req, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleAction.java
index c327a14fee236..10475b6940c0e 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleAction.java
@@ -37,11 +37,11 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        String[] lifecycleNames = Strings.splitStringByCommaToArray(request.param("name"));
-        GetSnapshotLifecycleAction.Request req = new GetSnapshotLifecycleAction.Request(lifecycleNames);
-        req.ackTimeout(getAckTimeout(request));
-        req.masterNodeTimeout(getMasterNodeTimeout(request));
-
+        final var req = new GetSnapshotLifecycleAction.Request(
+            getMasterNodeTimeout(request),
+            getAckTimeout(request),
+            Strings.splitStringByCommaToArray(request.param("name"))
+        );
         return channel -> client.execute(GetSnapshotLifecycleAction.INSTANCE, req, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestPutSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestPutSnapshotLifecycleAction.java
index 3db974a9af0a9..747ba60115ae7 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestPutSnapshotLifecycleAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestPutSnapshotLifecycleAction.java
@@ -38,11 +38,13 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        String snapLifecycleName = request.param("name");
         try (XContentParser parser = request.contentParser()) {
-            PutSnapshotLifecycleAction.Request req = PutSnapshotLifecycleAction.Request.parseRequest(snapLifecycleName, parser);
-            req.ackTimeout(getAckTimeout(request));
-            req.masterNodeTimeout(getMasterNodeTimeout(request));
+            final var req = PutSnapshotLifecycleAction.Request.parseRequest(
+                getMasterNodeTimeout(request),
+                getAckTimeout(request),
+                request.param("name"),
+                parser
+            );
             return channel -> client.execute(PutSnapshotLifecycleAction.INSTANCE, req, new RestToXContentListener<>(channel));
         }
     }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestStartSLMAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestStartSLMAction.java
index 3463346377f6c..9d1df9d72d2fe 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestStartSLMAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestStartSLMAction.java
@@ -36,9 +36,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        StartSLMAction.Request request = new StartSLMAction.Request();
-        request.ackTimeout(getAckTimeout(restRequest));
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        final var request = new StartSLMAction.Request(getMasterNodeTimeout(restRequest), getAckTimeout(restRequest));
         return channel -> client.execute(StartSLMAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestStopSLMAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestStopSLMAction.java
index 8c5d7296fac86..8a6280091b4fd 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestStopSLMAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/RestStopSLMAction.java
@@ -36,9 +36,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        StopSLMAction.Request request = new StopSLMAction.Request();
-        request.ackTimeout(getAckTimeout(restRequest));
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        final var request = new StopSLMAction.Request(getMasterNodeTimeout(restRequest), getAckTimeout(restRequest));
         return channel -> client.execute(StopSLMAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java
index 3b2dc9e23d172..062954e40d82c 100644
--- a/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java
+++ b/x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -84,14 +83,6 @@ public static class DeleteSnapshotPolicyTask extends AckedClusterStateUpdateTask
             this.request = request;
         }
 
-        /**
-         * Used by the {@link ReservedClusterStateHandler} for SLM
-         * {@link ReservedSnapshotAction}
-         */
-        DeleteSnapshotPolicyTask(String policyId) {
-            this(new DeleteSnapshotLifecycleAction.Request(policyId), null);
-        }
-
         @Override
         public ClusterState execute(ClusterState currentState) {
             SnapshotLifecycleMetadata snapMeta = currentState.metadata().custom(SnapshotLifecycleMetadata.TYPE);
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
index c2e3786a1afe7..1bb8e0183b2ea 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
@@ -377,7 +377,7 @@ public void testDeleteSLMReservedStateHandler() {
         );
         assertEquals(ReservedSnapshotAction.NAME, deleteAction.reservedStateHandlerName().get());
 
-        var request = new DeleteSnapshotLifecycleAction.Request("daily-snapshots1");
+        var request = new DeleteSnapshotLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "daily-snapshots1");
         assertThat(deleteAction.modifiedKeys(request), containsInAnyOrder("daily-snapshots1"));
     }
 
@@ -411,7 +411,12 @@ public void testPutSLMReservedStateHandler() throws Exception {
             }""";
 
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)) {
-            var request = PutSnapshotLifecycleAction.Request.parseRequest("daily-snapshots", parser);
+            var request = PutSnapshotLifecycleAction.Request.parseRequest(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                "daily-snapshots",
+                parser
+            );
 
             assertThat(putAction.modifiedKeys(request), containsInAnyOrder("daily-snapshots"));
         }
diff --git a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java
index 64ede24c9f65a..1c7b810de3d62 100644
--- a/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java
+++ b/x-pack/plugin/slm/src/test/java/org/elasticsearch/xpack/slm/action/TransportStopSLMActionTests.java
@@ -51,7 +51,7 @@ public void testStopILMClusterStatePriorityIsImmediate() {
             new TaskId(randomLong() + ":" + randomLong()),
             emptyMap()
         );
-        StopSLMAction.Request request = new StopSLMAction.Request();
+        StopSLMAction.Request request = new StopSLMAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         transportStopSLMAction.masterOperation(task, request, ClusterState.EMPTY_STATE, ActionListener.noop());
 
         verify(clusterService).submitUnbatchedStateUpdateTask(

From 7504fed0b3c8e78818ffa2014c16a2e285adf18d Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Tue, 28 May 2024 13:51:21 -0500
Subject: [PATCH 016/208] remote_cluster role documentation and expose to built
 in privs API  (#108840)

This commit introduces the documentation for remote_clusters which is used to help
 express the monitor_enrich privilege needed to use the ENRICH keyword across clusters
when using the API key based CCS security model.

This commit also adds "remote_clusters" to the built in privs API to for easier consumption
 in Kibana.
---
 .../security/get-builtin-privileges.asciidoc  |  8 +++-
 .../authorization/managing-roles.asciidoc     | 37 ++++++++++++++++++-
 .../GetBuiltinPrivilegesResponse.java         | 26 ++++++++++---
 .../TransportGetBuiltinPrivilegesAction.java  |  4 +-
 .../RestGetBuiltinPrivilegesAction.java       |  4 ++
 5 files changed, 70 insertions(+), 9 deletions(-)

diff --git a/docs/reference/rest-api/security/get-builtin-privileges.asciidoc b/docs/reference/rest-api/security/get-builtin-privileges.asciidoc
index 63f906d29b4d6..bbd0ca03c0473 100644
--- a/docs/reference/rest-api/security/get-builtin-privileges.asciidoc
+++ b/docs/reference/rest-api/security/get-builtin-privileges.asciidoc
@@ -45,6 +45,9 @@ version of {es}.
 <<privileges-list-indices,index privileges>> that are understood by this version
 of {es}.
 
+`remote_cluster`:: (array of string) The list of
+<<roles-remote-cluster-priv, remote_cluster>> privileges that are understood by this version
+of {es}.
 
 [[security-api-get-builtin-privileges-example]]
 ==== {api-examples-title}
@@ -56,7 +59,7 @@ The following example retrieves the names of all builtin privileges:
 GET /_security/privilege/_builtin
 --------------------------------------------------
 
-A successful call returns an object with "cluster" and "index" fields.
+A successful call returns an object with "cluster", "index", and "remote_cluster" fields.
 
 [source,console-result]
 --------------------------------------------------
@@ -145,6 +148,9 @@ A successful call returns an object with "cluster" and "index" fields.
     "read_cross_cluster",
     "view_index_metadata",
     "write"
+  ],
+  "remote_cluster" : [
+    "monitor_enrich"
   ]
 }
 --------------------------------------------------
diff --git a/docs/reference/security/authorization/managing-roles.asciidoc b/docs/reference/security/authorization/managing-roles.asciidoc
index a6197327db683..253aa33822234 100644
--- a/docs/reference/security/authorization/managing-roles.asciidoc
+++ b/docs/reference/security/authorization/managing-roles.asciidoc
@@ -12,7 +12,8 @@ A role is defined by the following JSON structure:
   "global": { ... }, <3>
   "indices": [ ... ], <4>
   "applications": [ ... ], <5>
-  "remote_indices": [ ... ] <6>
+  "remote_indices": [ ... ], <6>
+  "remote_cluster": [ ... ] <7>
 }
 -----
 // NOTCONSOLE
@@ -35,6 +36,10 @@ A role is defined by the following JSON structure:
     <<remote-clusters-api-key,remote clusters configured with the API key based model>>.
     This field is optional (missing `remote_indices` privileges effectively mean
     no index level permissions for any API key based remote clusters).
+<7> A list of cluster permissions entries for
+    <<remote-clusters-api-key,remote clusters configured with the API key based model>>.
+    This field is optional (missing `remote_cluster` privileges effectively means
+    no additional cluster permissions for any API key based remote clusters).
 
 [[valid-role-name]]
 NOTE: Role names must be at least 1 and no more than 507 characters. They can
@@ -209,6 +214,36 @@ The following describes the structure of a remote indices permissions entry:
     restricted indices, you must set this field to `true` (default is `false`), and then the
     `names` field will cover the restricted indices as well.
 
+[[roles-remote-cluster-priv]]
+==== Remote cluster privileges
+
+For <<remote-clusters-api-key,remote clusters configured with the API key based model>>, remote cluster privileges
+can be used to specify additional cluster privileges for matching remote clusters.
+
+NOTE: Remote cluster privileges are only effective for remote clusters configured with the API key based model.
+They have no effect on remote clusters configured with the certificate based model.
+
+The following describes the structure of a remote cluster permissions entry:
+
+[source,js]
+-------
+{
+  "clusters": [ ... ], <1>
+  "privileges": [ ... ] <2>
+}
+-------
+// NOTCONSOLE
+<1> A list of remote cluster aliases. It supports literal strings as well as
+<<api-multi-index,wildcards>> and <<regexp-syntax,regular expressions>>.
+This field is required.
+<2> The cluster level privileges for the remote cluster. The allowed values here are a subset of the
+<<privileges-list-cluster,cluster privileges>>. This field is required.
+
+The `monitor_enrich` privilege for remote clusters was introduced in version
+8.15.0. Currently, this is the only privilege available for remote clusters and
+is required to enable users to use the `ENRICH` keyword in ES|QL queries across
+clusters.
+
 ==== Example
 
 The following snippet shows an example definition of a `clicks_admin` role:
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponse.java
index 328089a73b2f5..27b7d48ea2cfb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponse.java
@@ -23,17 +23,27 @@ public final class GetBuiltinPrivilegesResponse extends ActionResponse {
 
     private final String[] clusterPrivileges;
     private final String[] indexPrivileges;
+    private final String[] remoteClusterPrivileges;
 
+    // used by serverless
     public GetBuiltinPrivilegesResponse(Collection<String> clusterPrivileges, Collection<String> indexPrivileges) {
-        this.clusterPrivileges = Objects.requireNonNull(
-            clusterPrivileges.toArray(Strings.EMPTY_ARRAY),
-            "Cluster privileges cannot be null"
-        );
-        this.indexPrivileges = Objects.requireNonNull(indexPrivileges.toArray(Strings.EMPTY_ARRAY), "Index privileges cannot be null");
+        this(clusterPrivileges, indexPrivileges, Collections.emptySet());
+    }
+
+    public GetBuiltinPrivilegesResponse(
+        Collection<String> clusterPrivileges,
+        Collection<String> indexPrivileges,
+        Collection<String> remoteClusterPrivileges
+    ) {
+        this.clusterPrivileges = Objects.requireNonNull(clusterPrivileges, "Cluster privileges cannot be null")
+            .toArray(Strings.EMPTY_ARRAY);
+        this.indexPrivileges = Objects.requireNonNull(indexPrivileges, "Index privileges cannot be null").toArray(Strings.EMPTY_ARRAY);
+        this.remoteClusterPrivileges = Objects.requireNonNull(remoteClusterPrivileges, "Remote cluster privileges cannot be null")
+            .toArray(Strings.EMPTY_ARRAY);
     }
 
     public GetBuiltinPrivilegesResponse() {
-        this(Collections.emptySet(), Collections.emptySet());
+        this(Collections.emptySet(), Collections.emptySet(), Collections.emptySet());
     }
 
     public String[] getClusterPrivileges() {
@@ -44,6 +54,10 @@ public String[] getIndexPrivileges() {
         return indexPrivileges;
     }
 
+    public String[] getRemoteClusterPrivileges() {
+        return remoteClusterPrivileges;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         TransportAction.localOnly();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java
index 8ea8ec3e0dcd9..62f0087c1a2d2 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 
@@ -34,7 +35,8 @@ public TransportGetBuiltinPrivilegesAction(ActionFilters actionFilters, Transpor
     protected void doExecute(Task task, GetBuiltinPrivilegesRequest request, ActionListener<GetBuiltinPrivilegesResponse> listener) {
         final TreeSet<String> cluster = new TreeSet<>(ClusterPrivilegeResolver.names());
         final TreeSet<String> index = new TreeSet<>(IndexPrivilege.names());
-        listener.onResponse(new GetBuiltinPrivilegesResponse(cluster, index));
+        final TreeSet<String> remoteCluster = new TreeSet<>(RemoteClusterPermissions.getSupportedRemoteClusterPermissions());
+        listener.onResponse(new GetBuiltinPrivilegesResponse(cluster, index, remoteCluster));
     }
 
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java
index 334e560312db1..d804559bba0ec 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java
@@ -73,6 +73,10 @@ public RestResponse buildResponse(GetBuiltinPrivilegesResponse response, XConten
                     builder.startObject();
                     builder.array("cluster", translatedResponse.getClusterPrivileges());
                     builder.array("index", translatedResponse.getIndexPrivileges());
+                    String[] remoteClusterPrivileges = translatedResponse.getRemoteClusterPrivileges();
+                    if (remoteClusterPrivileges.length > 0) { // remote clusters are not supported in stateless mode, so hide entirely
+                        builder.array("remote_cluster", remoteClusterPrivileges);
+                    }
                     builder.endObject();
                     return new RestResponse(RestStatus.OK, builder);
                 }

From 96075deff3806f7b7e7c0c07696a0ee44d59e2a5 Mon Sep 17 00:00:00 2001
From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com>
Date: Tue, 28 May 2024 15:15:26 -0400
Subject: [PATCH 017/208] [ML] Refactoring http settings and adding stats
 endpoint (#108219)

* Refactoring http settings and adding comments

* Removing new line

* Adding stats endpoint

* Improving comment and adding tests

* Fixing typo in setting name

* Switching to TransportNodesAction

* Fixing test

* Trying to fix test

* Adding equals hashcode

* Trying to send to all nodes

* Renaming to .diagnostics

* Removing unneeded variable

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../action/GetInferenceDiagnosticsAction.java | 254 ++++++++++++++++++
 ...nceDiagnosticsActionNodeResponseTests.java |  85 ++++++
 ...ferenceDiagnosticsActionResponseTests.java |  70 +++++
 .../xpack/inference/InferencePlugin.java      |  18 +-
 ...ransportGetInferenceDiagnosticsAction.java |  78 ++++++
 .../xpack/inference/common/Truncator.java     |   2 +-
 .../inference/external/http/HttpClient.java   |  19 ++
 .../external/http/HttpClientManager.java      |  94 ++++---
 .../inference/external/http/HttpSettings.java |   5 +-
 .../xpack/inference/rest/Paths.java           |   1 +
 .../RestGetInferenceDiagnosticsAction.java    |  44 +++
 .../external/http/HttpClientManagerTests.java |   2 +-
 .../xpack/security/operator/Constants.java    |   1 +
 13 files changed, 629 insertions(+), 44 deletions(-)
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsAction.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsActionNodeResponseTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsActionResponseTests.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceDiagnosticsAction.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestGetInferenceDiagnosticsAction.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsAction.java
new file mode 100644
index 0000000000000..29edc88ecda70
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsAction.java
@@ -0,0 +1,254 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.inference.action;
+
+import org.apache.http.pool.PoolStats;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.action.support.nodes.BaseNodeResponse;
+import org.elasticsearch.action.support.nodes.BaseNodesRequest;
+import org.elasticsearch.action.support.nodes.BaseNodesResponse;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public class GetInferenceDiagnosticsAction extends ActionType<GetInferenceDiagnosticsAction.Response> {
+
+    public static final GetInferenceDiagnosticsAction INSTANCE = new GetInferenceDiagnosticsAction();
+    public static final String NAME = "cluster:monitor/xpack/inference/diagnostics/get";
+
+    public GetInferenceDiagnosticsAction() {
+        super(NAME);
+    }
+
+    public static class Request extends BaseNodesRequest<Request> {
+
+        public Request() {
+            super((String[]) null);
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            return true;
+        }
+
+        @Override
+        public int hashCode() {
+            // The class doesn't have any members at the moment so return the same hash code
+            return Objects.hash(NAME);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) {
+            TransportAction.localOnly();
+        }
+    }
+
+    public static class NodeRequest extends TransportRequest {
+        public NodeRequest(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        public NodeRequest() {}
+    }
+
+    public static class Response extends BaseNodesResponse<NodeResponse> implements Writeable, ToXContentObject {
+
+        public Response(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        public Response(ClusterName clusterName, List<NodeResponse> nodes, List<FailedNodeException> failures) {
+            super(clusterName, nodes, failures);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+
+            for (var entry : getNodesMap().entrySet()) {
+                NodeResponse response = entry.getValue();
+
+                builder.startObject(entry.getKey());
+                response.toXContent(builder, params);
+                builder.endObject();
+            }
+
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        protected List<NodeResponse> readNodesFrom(StreamInput in) throws IOException {
+            return in.readCollectionAsList(NodeResponse::new);
+        }
+
+        @Override
+        protected void writeNodesTo(StreamOutput out, List<NodeResponse> nodes) throws IOException {
+            out.writeCollection(nodes);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Response that = (Response) o;
+            return Objects.equals(getNodes(), that.getNodes()) && Objects.equals(failures(), that.failures());
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(getNodes(), failures());
+        }
+    }
+
+    public static class NodeResponse extends BaseNodeResponse implements ToXContentFragment {
+        static final String CONNECTION_POOL_STATS_FIELD_NAME = "connection_pool_stats";
+
+        private final ConnectionPoolStats connectionPoolStats;
+
+        public NodeResponse(DiscoveryNode node, PoolStats poolStats) {
+            super(node);
+            connectionPoolStats = ConnectionPoolStats.of(poolStats);
+        }
+
+        public NodeResponse(StreamInput in) throws IOException {
+            super(in);
+
+            connectionPoolStats = new ConnectionPoolStats(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            connectionPoolStats.writeTo(out);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(CONNECTION_POOL_STATS_FIELD_NAME, connectionPoolStats, params);
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            NodeResponse response = (NodeResponse) o;
+            return Objects.equals(connectionPoolStats, response.connectionPoolStats);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(connectionPoolStats);
+        }
+
+        ConnectionPoolStats getConnectionPoolStats() {
+            return connectionPoolStats;
+        }
+
+        static class ConnectionPoolStats implements ToXContentObject, Writeable {
+            static final String LEASED_CONNECTIONS = "leased_connections";
+            static final String PENDING_CONNECTIONS = "pending_connections";
+            static final String AVAILABLE_CONNECTIONS = "available_connections";
+            static final String MAX_CONNECTIONS = "max_connections";
+
+            static ConnectionPoolStats of(PoolStats poolStats) {
+                return new ConnectionPoolStats(poolStats.getLeased(), poolStats.getPending(), poolStats.getAvailable(), poolStats.getMax());
+            }
+
+            private final int leasedConnections;
+            private final int pendingConnections;
+            private final int availableConnections;
+            private final int maxConnections;
+
+            ConnectionPoolStats(int leasedConnections, int pendingConnections, int availableConnections, int maxConnections) {
+                this.leasedConnections = leasedConnections;
+                this.pendingConnections = pendingConnections;
+                this.availableConnections = availableConnections;
+                this.maxConnections = maxConnections;
+            }
+
+            ConnectionPoolStats(StreamInput in) throws IOException {
+                this.leasedConnections = in.readVInt();
+                this.pendingConnections = in.readVInt();
+                this.availableConnections = in.readVInt();
+                this.maxConnections = in.readVInt();
+            }
+
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                builder.startObject();
+                builder.field(LEASED_CONNECTIONS, leasedConnections);
+                builder.field(PENDING_CONNECTIONS, pendingConnections);
+                builder.field(AVAILABLE_CONNECTIONS, availableConnections);
+                builder.field(MAX_CONNECTIONS, maxConnections);
+                builder.endObject();
+
+                return builder;
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+                out.writeVInt(leasedConnections);
+                out.writeVInt(pendingConnections);
+                out.writeVInt(availableConnections);
+                out.writeVInt(maxConnections);
+            }
+
+            @Override
+            public boolean equals(Object o) {
+                if (this == o) return true;
+                if (o == null || getClass() != o.getClass()) return false;
+                ConnectionPoolStats that = (ConnectionPoolStats) o;
+                return leasedConnections == that.leasedConnections
+                    && pendingConnections == that.pendingConnections
+                    && availableConnections == that.availableConnections
+                    && maxConnections == that.maxConnections;
+            }
+
+            @Override
+            public int hashCode() {
+                return Objects.hash(leasedConnections, pendingConnections, availableConnections, maxConnections);
+            }
+
+            int getLeasedConnections() {
+                return leasedConnections;
+            }
+
+            int getPendingConnections() {
+                return pendingConnections;
+            }
+
+            int getAvailableConnections() {
+                return availableConnections;
+            }
+
+            int getMaxConnections() {
+                return maxConnections;
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsActionNodeResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsActionNodeResponseTests.java
new file mode 100644
index 0000000000000..a21354eb5a73d
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsActionNodeResponseTests.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.inference.action;
+
+import org.apache.http.pool.PoolStats;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+
+public class GetInferenceDiagnosticsActionNodeResponseTests extends AbstractWireSerializingTestCase<
+    GetInferenceDiagnosticsAction.NodeResponse> {
+    public static GetInferenceDiagnosticsAction.NodeResponse createRandom() {
+        DiscoveryNode node = DiscoveryNodeUtils.create("id");
+        var randomPoolStats = new PoolStats(randomInt(), randomInt(), randomInt(), randomInt());
+
+        return new GetInferenceDiagnosticsAction.NodeResponse(node, randomPoolStats);
+    }
+
+    @Override
+    protected Writeable.Reader<GetInferenceDiagnosticsAction.NodeResponse> instanceReader() {
+        return GetInferenceDiagnosticsAction.NodeResponse::new;
+    }
+
+    @Override
+    protected GetInferenceDiagnosticsAction.NodeResponse createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected GetInferenceDiagnosticsAction.NodeResponse mutateInstance(GetInferenceDiagnosticsAction.NodeResponse instance)
+        throws IOException {
+        var select = randomIntBetween(0, 3);
+        var connPoolStats = instance.getConnectionPoolStats();
+
+        return switch (select) {
+            case 0 -> new GetInferenceDiagnosticsAction.NodeResponse(
+                instance.getNode(),
+                new PoolStats(
+                    randomInt(),
+                    connPoolStats.getPendingConnections(),
+                    connPoolStats.getAvailableConnections(),
+                    connPoolStats.getMaxConnections()
+                )
+            );
+            case 1 -> new GetInferenceDiagnosticsAction.NodeResponse(
+                instance.getNode(),
+                new PoolStats(
+                    connPoolStats.getLeasedConnections(),
+                    randomInt(),
+                    connPoolStats.getAvailableConnections(),
+                    connPoolStats.getMaxConnections()
+                )
+            );
+            case 2 -> new GetInferenceDiagnosticsAction.NodeResponse(
+                instance.getNode(),
+                new PoolStats(
+                    connPoolStats.getLeasedConnections(),
+                    connPoolStats.getPendingConnections(),
+                    randomInt(),
+                    connPoolStats.getMaxConnections()
+                )
+            );
+            case 3 -> new GetInferenceDiagnosticsAction.NodeResponse(
+                instance.getNode(),
+                new PoolStats(
+                    connPoolStats.getLeasedConnections(),
+                    connPoolStats.getPendingConnections(),
+                    connPoolStats.getAvailableConnections(),
+                    randomInt()
+                )
+            );
+            default -> throw new UnsupportedEncodingException(Strings.format("Encountered unsupported case %s", select));
+        };
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsActionResponseTests.java
new file mode 100644
index 0000000000000..e3eb42efdc791
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsActionResponseTests.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.inference.action;
+
+import org.apache.http.pool.PoolStats;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.hamcrest.CoreMatchers;
+
+import java.io.IOException;
+import java.util.List;
+
+public class GetInferenceDiagnosticsActionResponseTests extends AbstractWireSerializingTestCase<GetInferenceDiagnosticsAction.Response> {
+
+    public static GetInferenceDiagnosticsAction.Response createRandom() {
+        List<GetInferenceDiagnosticsAction.NodeResponse> responses = randomList(
+            2,
+            10,
+            GetInferenceDiagnosticsActionNodeResponseTests::createRandom
+        );
+        return new GetInferenceDiagnosticsAction.Response(ClusterName.DEFAULT, responses, List.of());
+    }
+
+    public void testToXContent() throws IOException {
+        var node = DiscoveryNodeUtils.create("id");
+        var poolStats = new PoolStats(1, 2, 3, 4);
+        var entity = new GetInferenceDiagnosticsAction.Response(
+            ClusterName.DEFAULT,
+            List.of(new GetInferenceDiagnosticsAction.NodeResponse(node, poolStats)),
+            List.of()
+        );
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = org.elasticsearch.common.Strings.toString(builder);
+
+        assertThat(xContentResult, CoreMatchers.is("""
+            {"id":{"connection_pool_stats":{"leased_connections":1,"pending_connections":2,"available_connections":3,""" + """
+            "max_connections":4}}}"""));
+    }
+
+    @Override
+    protected Writeable.Reader<GetInferenceDiagnosticsAction.Response> instanceReader() {
+        return GetInferenceDiagnosticsAction.Response::new;
+    }
+
+    @Override
+    protected GetInferenceDiagnosticsAction.Response createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected GetInferenceDiagnosticsAction.Response mutateInstance(GetInferenceDiagnosticsAction.Response instance) {
+        return new GetInferenceDiagnosticsAction.Response(
+            ClusterName.DEFAULT,
+            instance.getNodes().subList(1, instance.getNodes().size()),
+            List.of()
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
index 83fc7323eab40..108da3beef789 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
@@ -39,10 +39,12 @@
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.inference.action.DeleteInferenceModelAction;
+import org.elasticsearch.xpack.core.inference.action.GetInferenceDiagnosticsAction;
 import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.action.PutInferenceModelAction;
 import org.elasticsearch.xpack.inference.action.TransportDeleteInferenceModelAction;
+import org.elasticsearch.xpack.inference.action.TransportGetInferenceDiagnosticsAction;
 import org.elasticsearch.xpack.inference.action.TransportGetInferenceModelAction;
 import org.elasticsearch.xpack.inference.action.TransportInferenceAction;
 import org.elasticsearch.xpack.inference.action.TransportInferenceUsageAction;
@@ -59,6 +61,7 @@
 import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
 import org.elasticsearch.xpack.inference.rest.RestDeleteInferenceModelAction;
+import org.elasticsearch.xpack.inference.rest.RestGetInferenceDiagnosticsAction;
 import org.elasticsearch.xpack.inference.rest.RestGetInferenceModelAction;
 import org.elasticsearch.xpack.inference.rest.RestInferenceAction;
 import org.elasticsearch.xpack.inference.rest.RestPutInferenceModelAction;
@@ -123,7 +126,8 @@ public InferencePlugin(Settings settings) {
             new ActionHandler<>(GetInferenceModelAction.INSTANCE, TransportGetInferenceModelAction.class),
             new ActionHandler<>(PutInferenceModelAction.INSTANCE, TransportPutInferenceModelAction.class),
             new ActionHandler<>(DeleteInferenceModelAction.INSTANCE, TransportDeleteInferenceModelAction.class),
-            new ActionHandler<>(XPackUsageFeatureAction.INFERENCE, TransportInferenceUsageAction.class)
+            new ActionHandler<>(XPackUsageFeatureAction.INFERENCE, TransportInferenceUsageAction.class),
+            new ActionHandler<>(GetInferenceDiagnosticsAction.INSTANCE, TransportGetInferenceDiagnosticsAction.class)
         );
     }
 
@@ -143,7 +147,8 @@ public List<RestHandler> getRestHandlers(
             new RestInferenceAction(),
             new RestGetInferenceModelAction(),
             new RestPutInferenceModelAction(),
-            new RestDeleteInferenceModelAction()
+            new RestDeleteInferenceModelAction(),
+            new RestGetInferenceDiagnosticsAction()
         );
     }
 
@@ -153,11 +158,8 @@ public Collection<?> createComponents(PluginServices services) {
         var truncator = new Truncator(settings, services.clusterService());
         serviceComponents.set(new ServiceComponents(services.threadPool(), throttlerManager, settings, truncator));
 
-        var httpRequestSenderFactory = new HttpRequestSender.Factory(
-            serviceComponents.get(),
-            HttpClientManager.create(settings, services.threadPool(), services.clusterService(), throttlerManager),
-            services.clusterService()
-        );
+        var httpClientManager = HttpClientManager.create(settings, services.threadPool(), services.clusterService(), throttlerManager);
+        var httpRequestSenderFactory = new HttpRequestSender.Factory(serviceComponents.get(), httpClientManager, services.clusterService());
         httpFactory.set(httpRequestSenderFactory);
 
         ModelRegistry modelRegistry = new ModelRegistry(services.client());
@@ -178,7 +180,7 @@ public Collection<?> createComponents(PluginServices services) {
         var actionFilter = new ShardBulkInferenceActionFilter(registry, modelRegistry);
         shardBulkInferenceActionFilter.set(actionFilter);
 
-        return List.of(modelRegistry, registry);
+        return List.of(modelRegistry, registry, httpClientManager);
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceDiagnosticsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceDiagnosticsAction.java
new file mode 100644
index 0000000000000..8c9ab8f4cdffa
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceDiagnosticsAction.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.action;
+
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.nodes.TransportNodesAction;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.inference.action.GetInferenceDiagnosticsAction;
+import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public class TransportGetInferenceDiagnosticsAction extends TransportNodesAction<
+    GetInferenceDiagnosticsAction.Request,
+    GetInferenceDiagnosticsAction.Response,
+    GetInferenceDiagnosticsAction.NodeRequest,
+    GetInferenceDiagnosticsAction.NodeResponse> {
+
+    private final HttpClientManager httpClientManager;
+
+    @Inject
+    public TransportGetInferenceDiagnosticsAction(
+        ThreadPool threadPool,
+        ClusterService clusterService,
+        TransportService transportService,
+        ActionFilters actionFilters,
+        HttpClientManager httpClientManager
+    ) {
+        super(
+            GetInferenceDiagnosticsAction.NAME,
+            clusterService,
+            transportService,
+            actionFilters,
+            GetInferenceDiagnosticsAction.NodeRequest::new,
+            threadPool.executor(ThreadPool.Names.MANAGEMENT)
+        );
+
+        this.httpClientManager = Objects.requireNonNull(httpClientManager);
+    }
+
+    @Override
+    protected GetInferenceDiagnosticsAction.Response newResponse(
+        GetInferenceDiagnosticsAction.Request request,
+        List<GetInferenceDiagnosticsAction.NodeResponse> nodeResponses,
+        List<FailedNodeException> failures
+    ) {
+        return new GetInferenceDiagnosticsAction.Response(clusterService.getClusterName(), nodeResponses, failures);
+    }
+
+    @Override
+    protected GetInferenceDiagnosticsAction.NodeRequest newNodeRequest(GetInferenceDiagnosticsAction.Request request) {
+        return new GetInferenceDiagnosticsAction.NodeRequest();
+    }
+
+    @Override
+    protected GetInferenceDiagnosticsAction.NodeResponse newNodeResponse(StreamInput in, DiscoveryNode node) throws IOException {
+        return new GetInferenceDiagnosticsAction.NodeResponse(in);
+    }
+
+    @Override
+    protected GetInferenceDiagnosticsAction.NodeResponse nodeOperation(GetInferenceDiagnosticsAction.NodeRequest request, Task task) {
+        return new GetInferenceDiagnosticsAction.NodeResponse(transportService.getLocalNode(), httpClientManager.getPoolStats());
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/Truncator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/Truncator.java
index 2da509d0d9520..eabed7f6a7bd3 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/Truncator.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/Truncator.java
@@ -26,7 +26,7 @@ public class Truncator {
      * Defines the percentage to reduce the input text for an inference request.
      */
     static final Setting<Double> REDUCTION_PERCENTAGE_SETTING = Setting.doubleSetting(
-        "xpack.inference.truncator.reducation_percentage",
+        "xpack.inference.truncator.reduction_percentage",
         0.5,
         0.01,
         0.99,
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java
index 99631c380b9fa..5ae137419b366 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java
@@ -66,6 +66,25 @@ private static CloseableHttpAsyncClient createAsyncClient(PoolingNHttpClientConn
         // so we don't want to support cookies to avoid accidental authentication for unauthorized users
         clientBuilder.disableCookieManagement();
 
+        /*
+          By default, if a keep-alive header is not returned by the server then the connection will be kept alive
+          indefinitely. In this situation the default keep alive strategy will return -1. Since we use a connection eviction thread,
+          connections that are idle past the max idle time will be closed with the eviction thread executes. If that functionality proves
+          not to be sufficient we can add a keep-alive strategy to the builder below.
+
+          In my testing, setting a keep-alive didn't actually influence when the connection would be removed from the pool. Setting a low
+          keep alive forced later requests that occurred after the duration to recreate the connection. The stale connections would not be
+          removed from the pool until the eviction thread closes expired connections.
+
+          My understanding is that a connection marked as ready to be closed because of an elapsed keep-alive time will only be put into
+          expiry status when another request is made.
+
+          For more info see the tutorial here under section keep-alive strategy:
+          https://hc.apache.org/httpcomponents-client-4.5.x/current/tutorial/html/connmgmt.html
+
+          And this stackoverflow question:
+          https://stackoverflow.com/questions/64676200/understanding-the-lifecycle-of-a-connection-managed-by-poolinghttpclientconnecti
+         */
         return clientBuilder.build();
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClientManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClientManager.java
index ab3a8a8c0e043..8be3b76f68c54 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClientManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClientManager.java
@@ -12,6 +12,7 @@
 import org.apache.http.impl.nio.reactor.IOReactorConfig;
 import org.apache.http.nio.reactor.ConnectingIOReactor;
 import org.apache.http.nio.reactor.IOReactorException;
+import org.apache.http.pool.PoolStats;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
@@ -31,12 +32,24 @@
 public class HttpClientManager implements Closeable {
     private static final Logger logger = LogManager.getLogger(HttpClientManager.class);
     /**
+     * The maximum number of total connections the connection pool can lease to all routes.
      * From googling around the connection pools maxTotal value should be close to the number of available threads.
      *
      * https://stackoverflow.com/questions/30989637/how-to-decide-optimal-settings-for-setmaxtotal-and-setdefaultmaxperroute
      */
-    public static final Setting<Integer> MAX_CONNECTIONS = Setting.intSetting(
-        "xpack.inference.http.max_connections",
+    public static final Setting<Integer> MAX_TOTAL_CONNECTIONS = Setting.intSetting(
+        "xpack.inference.http.max_total_connections",
+        50, // default
+        1, // min
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    /**
+     * The max number of connections a single route can lease.
+     */
+    public static final Setting<Integer> MAX_ROUTE_CONNECTIONS = Setting.intSetting(
+        "xpack.inference.http.max_route_connections",
         20, // default
         1, // min
         Setting.Property.NodeScope,
@@ -51,20 +64,30 @@ public class HttpClientManager implements Closeable {
         Setting.Property.Dynamic
     );
 
-    private static final TimeValue DEFAULT_CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING = DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME;
-    public static final Setting<TimeValue> CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING = Setting.timeSetting(
+    private static final TimeValue DEFAULT_CONNECTION_MAX_IDLE_TIME_SETTING = DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME;
+    /**
+     * The max duration of time for a connection to be marked as idle and ready to be closed. This defines the amount of time
+     * a connection can be unused in the connection pool before being closed the next time the eviction thread runs.
+     * It also defines the keep-alive value for the connection if one is not specified by the 3rd party service's server.
+     *
+     * For more info see the answer here:
+     * https://stackoverflow.com/questions/64676200/understanding-the-lifecycle-of-a-connection-managed-by-poolinghttpclientconnecti
+     */
+    public static final Setting<TimeValue> CONNECTION_MAX_IDLE_TIME_SETTING = Setting.timeSetting(
         "xpack.inference.http.connection_eviction_max_idle_time",
-        DEFAULT_CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING,
+        DEFAULT_CONNECTION_MAX_IDLE_TIME_SETTING,
         Setting.Property.NodeScope,
         Setting.Property.Dynamic
     );
 
     private final ThreadPool threadPool;
     private final PoolingNHttpClientConnectionManager connectionManager;
-    private EvictorSettings evictorSettings;
     private IdleConnectionEvictor connectionEvictor;
     private final HttpClient httpClient;
 
+    private volatile TimeValue evictionInterval;
+    private volatile TimeValue connectionMaxIdle;
+
     public static HttpClientManager create(
         Settings settings,
         ThreadPool threadPool,
@@ -86,11 +109,13 @@ public static HttpClientManager create(
         this.threadPool = threadPool;
 
         this.connectionManager = connectionManager;
-        setMaxConnections(MAX_CONNECTIONS.get(settings));
+        setMaxConnections(MAX_TOTAL_CONNECTIONS.get(settings));
+        setMaxRouteConnections(MAX_ROUTE_CONNECTIONS.get(settings));
 
         this.httpClient = HttpClient.create(new HttpSettings(settings, clusterService), threadPool, connectionManager, throttlerManager);
 
-        evictorSettings = new EvictorSettings(settings);
+        this.evictionInterval = CONNECTION_EVICTION_THREAD_INTERVAL_SETTING.get(settings);
+        this.connectionMaxIdle = CONNECTION_MAX_IDLE_TIME_SETTING.get(settings);
         connectionEvictor = createConnectionEvictor();
 
         this.addSettingsUpdateConsumers(clusterService);
@@ -107,22 +132,34 @@ private static PoolingNHttpClientConnectionManager createConnectionManager() {
             throw new ElasticsearchException(message, e);
         }
 
+        /*
+          The max time to live for open connections in the pool will not be set because we don't specify a ttl in the constructor.
+          This meaning that there should not be a limit.
+          We can control the TTL dynamically using the IdleConnectionEvictor and keep-alive strategy.
+          The max idle time cluster setting will dictate how much time an open connection can be unused for before it can be closed.
+         */
         return new PoolingNHttpClientConnectionManager(ioReactor);
     }
 
     private void addSettingsUpdateConsumers(ClusterService clusterService) {
-        clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_CONNECTIONS, this::setMaxConnections);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_TOTAL_CONNECTIONS, this::setMaxConnections);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_ROUTE_CONNECTIONS, this::setMaxRouteConnections);
         clusterService.getClusterSettings()
             .addSettingsUpdateConsumer(CONNECTION_EVICTION_THREAD_INTERVAL_SETTING, this::setEvictionInterval);
-        clusterService.getClusterSettings().addSettingsUpdateConsumer(CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING, this::setEvictionMaxIdle);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(CONNECTION_MAX_IDLE_TIME_SETTING, this::setConnectionMaxIdle);
     }
 
     private IdleConnectionEvictor createConnectionEvictor() {
-        return new IdleConnectionEvictor(threadPool, connectionManager, evictorSettings.evictionInterval, evictorSettings.evictionMaxIdle);
+        return new IdleConnectionEvictor(threadPool, connectionManager, evictionInterval, connectionMaxIdle);
     }
 
     public static List<Setting<?>> getSettings() {
-        return List.of(MAX_CONNECTIONS, CONNECTION_EVICTION_THREAD_INTERVAL_SETTING, CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING);
+        return List.of(
+            MAX_TOTAL_CONNECTIONS,
+            MAX_ROUTE_CONNECTIONS,
+            CONNECTION_EVICTION_THREAD_INTERVAL_SETTING,
+            CONNECTION_MAX_IDLE_TIME_SETTING
+        );
     }
 
     public void start() {
@@ -134,6 +171,10 @@ public HttpClient getHttpClient() {
         return httpClient;
     }
 
+    public PoolStats getPoolStats() {
+        return connectionManager.getTotalStats();
+    }
+
     @Override
     public void close() throws IOException {
         httpClient.close();
@@ -142,6 +183,9 @@ public void close() throws IOException {
 
     private void setMaxConnections(int maxConnections) {
         connectionManager.setMaxTotal(maxConnections);
+    }
+
+    private void setMaxRouteConnections(int maxConnections) {
         connectionManager.setDefaultMaxPerRoute(maxConnections);
     }
 
@@ -153,32 +197,16 @@ boolean isEvictionThreadRunning() {
     // default for testing
     void setEvictionInterval(TimeValue evictionInterval) {
         logger.debug(() -> format("Eviction thread's interval time updated to [%s]", evictionInterval));
-
-        evictorSettings = new EvictorSettings(evictionInterval, evictorSettings.evictionMaxIdle);
+        this.evictionInterval = evictionInterval;
 
         connectionEvictor.close();
         connectionEvictor = createConnectionEvictor();
         connectionEvictor.start();
     }
 
-    void setEvictionMaxIdle(TimeValue evictionMaxIdle) {
-        logger.debug(() -> format("Eviction thread's max idle time updated to [%s]", evictionMaxIdle));
-        evictorSettings = new EvictorSettings(evictorSettings.evictionInterval, evictionMaxIdle);
-        connectionEvictor.setMaxIdleTime(evictionMaxIdle);
-    }
-
-    private static class EvictorSettings {
-        private final TimeValue evictionInterval;
-        private final TimeValue evictionMaxIdle;
-
-        EvictorSettings(Settings settings) {
-            this.evictionInterval = CONNECTION_EVICTION_THREAD_INTERVAL_SETTING.get(settings);
-            this.evictionMaxIdle = CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING.get(settings);
-        }
-
-        EvictorSettings(TimeValue evictionInterval, TimeValue evictionMaxIdle) {
-            this.evictionInterval = evictionInterval;
-            this.evictionMaxIdle = evictionMaxIdle;
-        }
+    void setConnectionMaxIdle(TimeValue connectionMaxIdle) {
+        logger.debug(() -> format("Eviction thread's max idle time updated to [%s]", connectionMaxIdle));
+        this.connectionMaxIdle = connectionMaxIdle;
+        connectionEvictor.setMaxIdleTime(connectionMaxIdle);
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java
index 07d998dff956e..ef5fec24c3d59 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 
 import java.util.List;
+import java.util.Objects;
 
 public class HttpSettings {
     // These settings are default scope for testing
@@ -29,7 +30,9 @@ public class HttpSettings {
     private volatile ByteSizeValue maxResponseSize;
 
     public HttpSettings(Settings settings, ClusterService clusterService) {
-        this.maxResponseSize = MAX_HTTP_RESPONSE_SIZE.get(settings);
+        Objects.requireNonNull(clusterService);
+        Objects.requireNonNull(settings);
+        maxResponseSize = MAX_HTTP_RESPONSE_SIZE.get(settings);
 
         clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_HTTP_RESPONSE_SIZE, this::setMaxResponseSize);
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java
index 1fc67d379a703..e33931f3d2f8d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/Paths.java
@@ -13,6 +13,7 @@ public final class Paths {
     static final String TASK_TYPE_OR_INFERENCE_ID = "task_type_or_id";
     static final String INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}";
     static final String TASK_TYPE_INFERENCE_ID_PATH = "_inference/{" + TASK_TYPE_OR_INFERENCE_ID + "}/{" + INFERENCE_ID + "}";
+    static final String INFERENCE_DIAGNOSTICS_PATH = "_inference/.diagnostics";
 
     private Paths() {
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestGetInferenceDiagnosticsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestGetInferenceDiagnosticsAction.java
new file mode 100644
index 0000000000000..18ee2bc348bd3
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestGetInferenceDiagnosticsAction.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.rest;
+
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.inference.action.GetInferenceDiagnosticsAction;
+
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.xpack.inference.rest.Paths.INFERENCE_DIAGNOSTICS_PATH;
+
+@ServerlessScope(Scope.INTERNAL)
+public class RestGetInferenceDiagnosticsAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "get_inference_diagnostics_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(GET, INFERENCE_DIAGNOSTICS_PATH));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+        return channel -> client.execute(
+            GetInferenceDiagnosticsAction.INSTANCE,
+            new GetInferenceDiagnosticsAction.Request(),
+            new RestToXContentListener<>(channel)
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java
index cfdf2e8bb8bcd..7bd8186299522 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java
@@ -106,7 +106,7 @@ public void test_DoesNotStartANewEvictor_WithNewEvictionMaxIdle() {
         );
 
         var evictionMaxIdle = TimeValue.timeValueSeconds(1);
-        manager.setEvictionMaxIdle(evictionMaxIdle);
+        manager.setConnectionMaxIdle(evictionMaxIdle);
 
         assertFalse(manager.isEvictionThreadRunning());
     }
diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
index 28f768d13dbf1..ae5af54f078dd 100644
--- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
+++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
@@ -369,6 +369,7 @@ public class Constants {
         "cluster:monitor/xpack/esql/stats/dist",
         "cluster:monitor/xpack/inference",
         "cluster:monitor/xpack/inference/get",
+        "cluster:monitor/xpack/inference/diagnostics/get",
         "cluster:monitor/xpack/info",
         "cluster:monitor/xpack/info/aggregate_metric",
         "cluster:monitor/xpack/info/analytics",

From d880c76fad39333cae24ad2a3169b7495dcacb95 Mon Sep 17 00:00:00 2001
From: Mike Pellegrini <mike.pellegrini@elastic.co>
Date: Tue, 28 May 2024 16:14:08 -0400
Subject: [PATCH 018/208] Handle partial semantic_text updates submitted
 through the Update API (#108880)

Update TransportUpdateAction to remove stale inference results from _source when a semantic_text field with multiple source fields is partially updated through the Update API.
---
 .../action/update/TransportUpdateAction.java  |  99 ++-
 .../index/mapper/InferenceFieldMapper.java    |   9 +
 ...appingLookupInferenceFieldMapperTests.java |   5 +
 .../mapper/SemanticTextFieldMapper.java       |  24 +-
 .../inference/30_semantic_text_inference.yml  | 317 +--------
 .../60_semantic_text_inference_update.yml     | 612 ++++++++++++++++++
 6 files changed, 749 insertions(+), 317 deletions(-)
 create mode 100644 x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml

diff --git a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index ad297c14981aa..cd899d732e916 100644
--- a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.update;
 
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
@@ -27,6 +28,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.cluster.routing.IndexRouting;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardIterator;
@@ -36,13 +38,18 @@
 import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.index.mapper.InferenceFieldMapper;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Names;
@@ -179,7 +186,13 @@ protected void shardOperation(final UpdateRequest request, final ActionListener<
         final ShardId shardId = request.getShardId();
         final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         final IndexShard indexShard = indexService.getShard(shardId.getId());
-        final UpdateHelper.Result result = updateHelper.prepare(request, indexShard, threadPool::absoluteTimeInMillis);
+        final UpdateHelper.Result result = deleteInferenceResults(
+            request,
+            updateHelper.prepare(request, indexShard, threadPool::absoluteTimeInMillis),
+            indexService.getMetadata(),
+            indexShard.mapperService().mappingLookup()
+        );
+
         switch (result.getResponseResult()) {
             case CREATED -> {
                 IndexRequest upsertRequest = result.action();
@@ -333,4 +346,88 @@ private void handleUpdateFailureWithRetry(
         }
         listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause));
     }
+
+    /**
+     * <p>
+     * Delete stale inference results from the provided {@link UpdateHelper.Result} instance.
+     * </p>
+     * <p>
+     * We need to do this because when handling Bulk API requests (which the Update API generates), we assume any inference results present
+     * in source are up-to-date.
+     * We do this to support reindex and update by query use cases without re-generating inference results unnecessarily.
+     * </p>
+     *
+     * @param updateRequest The update request
+     * @param result The result generated using the update request
+     * @param indexMetadata The index metadata
+     * @param mappingLookup The index's mapping lookup
+     * @return A result with stale inference results removed from source
+     */
+    private static UpdateHelper.Result deleteInferenceResults(
+        UpdateRequest updateRequest,
+        UpdateHelper.Result result,
+        IndexMetadata indexMetadata,
+        MappingLookup mappingLookup
+    ) {
+        if (result.getResponseResult() != DocWriteResponse.Result.UPDATED) {
+            return result;
+        }
+
+        Map<String, InferenceFieldMetadata> inferenceFields = indexMetadata.getInferenceFields();
+        if (inferenceFields.isEmpty()) {
+            return result;
+        }
+
+        if (updateRequest.script() != null) {
+            throw new ElasticsearchStatusException(
+                "Cannot apply update with a script on indices that contain inference field(s)",
+                RestStatus.BAD_REQUEST
+            );
+        }
+
+        IndexRequest doc = updateRequest.doc();
+        if (doc == null) {
+            // No doc update, nothing to do
+            return result;
+        }
+
+        Map<String, Object> updateRequestSource = doc.sourceAsMap();
+        Map<String, Object> updatedSource = result.updatedSourceAsMap();
+        boolean updatedSourceModified = false;
+        for (var entry : inferenceFields.entrySet()) {
+            String inferenceFieldName = entry.getKey();
+            Mapper mapper = mappingLookup.getMapper(inferenceFieldName);
+
+            if (mapper instanceof InferenceFieldMapper inferenceFieldMapper) {
+                String[] sourceFields = entry.getValue().getSourceFields();
+                for (String sourceField : sourceFields) {
+                    if (sourceField.equals(inferenceFieldName) == false
+                        && XContentMapValues.extractValue(sourceField, updateRequestSource) != null) {
+                        // Replace the inference field's value with its original value (i.e. the user-specified value).
+                        // This has two important side effects:
+                        // - The inference field value will remain parsable by its mapper
+                        // - The inference results will be removed, forcing them to be re-generated downstream
+                        updatedSource.put(inferenceFieldName, inferenceFieldMapper.getOriginalValue(updatedSource));
+                        updatedSourceModified = true;
+                        break;
+                    }
+                }
+            } else {
+                throw new IllegalStateException(
+                    "Field [" + inferenceFieldName + "] is of type [ " + mapper.typeName() + "], which is not an inference field"
+                );
+            }
+        }
+
+        UpdateHelper.Result returnedResult = result;
+        if (updatedSourceModified) {
+            XContentType contentType = result.updateSourceContentType();
+            IndexRequest indexRequest = result.action();
+            indexRequest.source(updatedSource, contentType);
+
+            returnedResult = new UpdateHelper.Result(indexRequest, result.getResponseResult(), updatedSource, contentType);
+        }
+
+        return returnedResult;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/InferenceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/InferenceFieldMapper.java
index 2b0833c72021b..cab06dc8a4e35 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/InferenceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/InferenceFieldMapper.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.inference.InferenceService;
 
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -24,4 +25,12 @@ public interface InferenceFieldMapper {
      * @param sourcePaths The source path that populates the input for the field (before inference)
      */
     InferenceFieldMetadata getMetadata(Set<String> sourcePaths);
+
+    /**
+     * Get the field's original value (i.e. the value the user specified) from the provided source.
+     *
+     * @param sourceAsMap The source as a map
+     * @return The field's original value, or {@code null} if none was provided
+     */
+    Object getOriginalValue(Map<String, Object> sourceAsMap);
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java
index b13f6b79b0de5..5b636c985f695 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupInferenceFieldMapperTests.java
@@ -104,6 +104,11 @@ public InferenceFieldMetadata getMetadata(Set<String> sourcePaths) {
             return new InferenceFieldMetadata(name(), INFERENCE_ID, sourcePaths.toArray(new String[0]));
         }
 
+        @Override
+        public Object getOriginalValue(Map<String, Object> sourceAsMap) {
+            return null;
+        }
+
         @Override
         protected void parseCreateField(DocumentParserContext context) throws IOException {}
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
index 8324b121dfc4f..6874938f1e118 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexVersion;
@@ -65,6 +66,7 @@
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKS_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_ID_FIELD;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.TEXT_FIELD;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getOriginalTextFieldName;
@@ -188,6 +190,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
             return;
         }
+
         XContentLocation xContentLocation = parser.getTokenLocation();
         final SemanticTextField field;
         boolean isWithinLeaf = context.path().isWithinLeafObject();
@@ -197,6 +200,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         } finally {
             context.path().setWithinLeafObject(isWithinLeaf);
         }
+
         final String fullFieldName = fieldType().name();
         if (field.inference().inferenceId().equals(fieldType().getInferenceId()) == false) {
             throw new DocumentParsingException(
@@ -211,6 +215,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
                 )
             );
         }
+
         final SemanticTextFieldMapper mapper;
         if (fieldType().getModelSettings() == null) {
             context.path().remove();
@@ -241,6 +246,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             }
             mapper = this;
         }
+
         var chunksField = mapper.fieldType().getChunksField();
         var embeddingsField = mapper.fieldType().getEmbeddingsField();
         for (var chunk : field.inference().chunks()) {
@@ -276,6 +282,20 @@ public InferenceFieldMetadata getMetadata(Set<String> sourcePaths) {
         return new InferenceFieldMetadata(name(), fieldType().inferenceId, copyFields);
     }
 
+    @Override
+    public Object getOriginalValue(Map<String, Object> sourceAsMap) {
+        Object fieldValue = sourceAsMap.get(name());
+        if (fieldValue == null) {
+            return null;
+        } else if (fieldValue instanceof Map<?, ?> == false) {
+            // Don't try to further validate the non-map value, that will be handled when the source is fully parsed
+            return fieldValue;
+        }
+
+        Map<String, Object> fieldValueMap = XContentMapValues.nodeMapValue(fieldValue, "Field [" + name() + "]");
+        return XContentMapValues.extractValue(TEXT_FIELD, fieldValueMap);
+    }
+
     public static class SemanticTextFieldType extends SimpleMappedFieldType {
         private final String inferenceId;
         private final SemanticTextField.ModelSettings modelSettings;
@@ -284,14 +304,14 @@ public static class SemanticTextFieldType extends SimpleMappedFieldType {
 
         public SemanticTextFieldType(
             String name,
-            String modelId,
+            String inferenceId,
             SemanticTextField.ModelSettings modelSettings,
             ObjectMapper inferenceField,
             IndexVersion indexVersionCreated,
             Map<String, String> meta
         ) {
             super(name, false, false, false, TextSearchInfo.NONE, meta);
-            this.inferenceId = modelId;
+            this.inferenceId = inferenceId;
             this.modelSettings = modelSettings;
             this.inferenceField = inferenceField;
             this.indexVersionCreated = indexVersionCreated;
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
index d4769ac1a1af3..9987b43822cc0 100644
--- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml
@@ -17,6 +17,7 @@ setup:
             "task_settings": {
             }
           }
+
   - do:
       inference.put:
         task_type: text_embedding
@@ -214,126 +215,6 @@ setup:
   - length: { hits.hits.1.inner_hits.dense_field\.inference\.chunks.hits.hits.0.fields.dense_field\.inference\.chunks.0.text: 1 }
   - length: { hits.hits.1.inner_hits.dense_field\.inference\.chunks.hits.hits.1.fields.dense_field\.inference\.chunks.0.text: 1 }
 
-
-
----
-"Updating non semantic_text fields does not recalculate embeddings":
-    - do:
-        index:
-          index: test-index
-          id: doc_1
-          body:
-            sparse_field: "inference test"
-            dense_field: "another inference test"
-            non_inference_field: "non inference test"
-
-    - do:
-        get:
-          index: test-index
-          id: doc_1
-
-    - set: { _source.sparse_field.inference.chunks.0.embeddings: sparse_field_embedding }
-    - set: { _source.dense_field.inference.chunks.0.embeddings: dense_field_embedding }
-
-    - do:
-        update:
-          index: test-index
-          id: doc_1
-          body:
-            doc:
-              non_inference_field: "another non inference test"
-
-    - do:
-        get:
-          index: test-index
-          id: doc_1
-
-    - match:  { _source.sparse_field.text: "inference test" }
-    - match:  { _source.sparse_field.inference.chunks.0.text: "inference test" }
-    - match:  { _source.sparse_field.inference.chunks.0.embeddings: $sparse_field_embedding }
-    - match:  { _source.dense_field.text: "another inference test" }
-    - match:  { _source.dense_field.inference.chunks.0.text: "another inference test" }
-    - match:  { _source.dense_field.inference.chunks.0.embeddings: $dense_field_embedding }
-    - match:  { _source.non_inference_field: "another non inference test" }
-
----
-"Updating semantic_text fields recalculates embeddings":
-    - do:
-        index:
-          index: test-index
-          id: doc_1
-          body:
-            sparse_field: "inference test"
-            dense_field: "another inference test"
-            non_inference_field: "non inference test"
-
-    - do:
-        get:
-          index: test-index
-          id: doc_1
-
-    - match:  { _source.sparse_field.text: "inference test" }
-    - match:  { _source.sparse_field.inference.chunks.0.text: "inference test" }
-    - match:  { _source.dense_field.text: "another inference test" }
-    - match:  { _source.dense_field.inference.chunks.0.text: "another inference test" }
-    - match:  { _source.non_inference_field: "non inference test" }
-
-    - do:
-        bulk:
-          index: test-index
-          body:
-            - '{"update": {"_id": "doc_1"}}'
-            - '{"doc":{"sparse_field": "I am a test", "dense_field": "I am a teapot"}}'
-
-    - do:
-        get:
-          index: test-index
-          id: doc_1
-
-    - match:  { _source.sparse_field.text: "I am a test" }
-    - match:  { _source.sparse_field.inference.chunks.0.text: "I am a test" }
-    - match:  { _source.dense_field.text: "I am a teapot" }
-    - match:  { _source.dense_field.inference.chunks.0.text: "I am a teapot" }
-    - match:  { _source.non_inference_field: "non inference test" }
-
-    - do:
-        update:
-          index: test-index
-          id: doc_1
-          body:
-            doc:
-              sparse_field: "updated inference test"
-              dense_field: "another updated inference test"
-
-    - do:
-        get:
-          index: test-index
-          id: doc_1
-
-    - match:  { _source.sparse_field.text: "updated inference test" }
-    - match:  { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
-    - match:  { _source.dense_field.text: "another updated inference test" }
-    - match:  { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
-    - match:  { _source.non_inference_field: "non inference test" }
-
-    - do:
-        bulk:
-          index: test-index
-          body:
-            - '{"update": {"_id": "doc_1"}}'
-            - '{"doc":{"sparse_field": "bulk inference test", "dense_field": "bulk updated inference test"}}'
-
-    - do:
-        get:
-          index: test-index
-          id: doc_1
-
-    - match:  { _source.sparse_field.text: "bulk inference test" }
-    - match:  { _source.sparse_field.inference.chunks.0.text: "bulk inference test" }
-    - match:  { _source.dense_field.text: "bulk updated inference test" }
-    - match:  { _source.dense_field.inference.chunks.0.text: "bulk updated inference test" }
-    - match:  { _source.non_inference_field: "non inference test" }
-
 ---
 "Reindex works for semantic_text fields":
   - do:
@@ -426,27 +307,7 @@ setup:
           non_inference_field: "non inference test"
 
 ---
-"Updates with script are not allowed":
-  - do:
-      bulk:
-        index: test-index
-        body:
-          - '{"index": {"_id": "doc_1"}}'
-          - '{"doc":{"sparse_field": "I am a test", "dense_field": "I am a teapot"}}'
-
-  - do:
-      bulk:
-        index: test-index
-        body:
-          - '{"update": {"_id": "doc_1"}}'
-          - '{"script": "ctx._source.new_field = \"hello\"", "scripted_upsert": true}'
-
-  - match: { errors: true }
-  - match: { items.0.update.status: 400 }
-  - match: { items.0.update.error.reason: "Cannot apply update with a script on indices that contain [semantic_text] field(s)" }
-
----
-"semantic_text copy_to calculate inference for source fields":
+"semantic_text copy_to calculates embeddings for source fields":
   - do:
       indices.create:
         index: test-copy-to-index
@@ -486,78 +347,8 @@ setup:
   - match:  { _source.sparse_field.inference.chunks.2.text: "inference test" }
   - exists:   _source.sparse_field.inference.chunks.2.embeddings
 
-
 ---
-"semantic_text copy_to needs values for every source field for updates":
-  - do:
-      indices.create:
-        index: test-copy-to-index
-        body:
-          mappings:
-            properties:
-              sparse_field:
-                type: semantic_text
-                inference_id: sparse-inference-id
-              source_field:
-                type: text
-                copy_to: sparse_field
-              another_source_field:
-                type: text
-                copy_to: sparse_field
-
-  # Not every source field needed on creation
-  - do:
-      index:
-        index: test-copy-to-index
-        id: doc_1
-        body:
-          source_field: "a single source field provided"
-          sparse_field: "inference test"
-
-  # Every source field needed on bulk updates
-  - do:
-      bulk:
-        body:
-          - '{"update": {"_index": "test-copy-to-index", "_id": "doc_1"}}'
-          - '{"doc": {"source_field": "a single source field is kept as provided via bulk", "sparse_field": "updated inference test" }}'
-
-  - match: { items.0.update.status: 400 }
-  - match: { items.0.update.error.reason: "Field [another_source_field] must be specified on an update request to calculate inference for field [sparse_field]" }
-
-
----
-"Update works for now - but will be unsupported later to avoid dealing with missing semantic_text fields content or copy_to fields":
-  - do:
-      index:
-        index: test-index
-        id: doc_1
-        body:
-          sparse_field: "inference test"
-          dense_field: "another inference test"
-          non_inference_field: "non inference test"
-
-  - do:
-      update:
-        index: test-index
-        id: doc_1
-        body:
-          doc: { "sparse_field": "updated inference test", "dense_field": "another updated inference test", "non_inference_field": "updated non inference test" }
-
-  - do:
-      get:
-        index: test-index
-        id: doc_1
-
-  - match: { _source.sparse_field.text: "updated inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
-  - match: { _source.dense_field.text: "another updated inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
-  - match: { _source.non_inference_field: "updated non inference test" }
-
----
-"Calculates text expansion results for bulk updates - index":
+"Calculates embeddings for bulk operations - index":
   - do:
       bulk:
         body:
@@ -580,108 +371,6 @@ setup:
   - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
   - match: { _source.non_inference_field: "non inference test" }
 
----
-"Calculates text expansion results for bulk updates - update":
-  - do:
-      bulk:
-        body:
-          - '{"index": {"_index": "test-index", "_id": "doc_1"}}'
-          - '{"sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test"}'
-
-  - match: { errors: false }
-  - match: { items.0.index.result: "created" }
-
-  - do:
-      bulk:
-        body:
-          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
-          - '{"doc": { "sparse_field": "updated inference test", "dense_field": "another updated inference test", "non_inference_field": "updated non inference test" }}'
-
-  - match: { errors: false }
-  - match: { items.0.update.result: "updated" }
-
-  - do:
-      get:
-        index: test-index
-        id: doc_1
-
-  - match: { _source.sparse_field.text: "updated inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
-  - match: { _source.dense_field.text: "another updated inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
-  - match: { _source.non_inference_field: "updated non inference test" }
-
-  # Script update not supported
-  - do:
-      bulk:
-        body:
-          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
-          - '{"script": {"source": {"ctx.sparse_field": "updated inference test"}}}'
-
-  - match: { errors: true }
-  - match: { items.0.update.status: 400 }
-  - match: { items.0.update.error.reason: "Cannot apply update with a script on indices that contain [semantic_text] field(s)" }
-
----
-"Calculates text expansion results for bulk updates - upsert":
-  # Initial update fails
-  - do:
-      bulk:
-        body:
-          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
-          - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }}'
-
-  - match: { errors: true }
-  - match: { items.0.update.status: 404 }
-
-  # Update as upsert
-  - do:
-      bulk:
-        body:
-          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
-          - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }, "doc_as_upsert": true}'
-
-  - match: { errors: false }
-  - match: { items.0.update.result: "created" }
-
-  - do:
-      get:
-        index: test-index
-        id: doc_1
-
-  - match: { _source.sparse_field.text: "inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
-  - match: { _source.dense_field.text: "another inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
-  - match: { _source.non_inference_field: "non inference test" }
-
-  - do:
-      bulk:
-        body:
-          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
-          - '{"doc": { "sparse_field": "updated inference test", "dense_field": "another updated inference test", "non_inference_field": "updated non inference test" }, "doc_as_upsert": true}'
-
-  - match: { errors: false }
-  - match: { items.0.update.result: "updated" }
-
-  - do:
-      get:
-        index: test-index
-        id: doc_1
-
-  - match: { _source.sparse_field.text: "updated inference test" }
-  - exists: _source.sparse_field.inference.chunks.0.embeddings
-  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
-  - match: { _source.dense_field.text: "another updated inference test" }
-  - exists: _source.dense_field.inference.chunks.0.embeddings
-  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
-  - match: { _source.non_inference_field: "updated non inference test" }
-
-
 ---
 "Update by query picks up new semantic_text fields":
 
diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml
new file mode 100644
index 0000000000000..59ce439d954a2
--- /dev/null
+++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml
@@ -0,0 +1,612 @@
+setup:
+  - requires:
+      cluster_features: "gte_v8.15.0"
+      reason: semantic_text introduced in 8.15.0
+
+  - do:
+      inference.put:
+        task_type: sparse_embedding
+        inference_id: sparse-inference-id
+        body: >
+          {
+            "service": "test_service",
+            "service_settings": {
+              "model": "my_model",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      inference.put:
+        task_type: text_embedding
+        inference_id: dense-inference-id
+        body: >
+          {
+            "service": "text_embedding_test_service",
+            "service_settings": {
+              "model": "my_model",
+              "dimensions": 10,
+              "similarity": "cosine",
+              "api_key": "abc64"
+            },
+            "task_settings": {
+            }
+          }
+
+  - do:
+      indices.create:
+        index: test-index
+        body:
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              non_inference_field:
+                type: text
+
+---
+"Updating non semantic_text fields does not recalculate embeddings":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: "inference test"
+          dense_field: "another inference test"
+          non_inference_field: "non inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - set: { _source.sparse_field.inference.chunks.0.embeddings: sparse_field_embedding }
+  - set: { _source.dense_field.inference.chunks.0.embeddings: dense_field_embedding }
+
+  - do:
+      update:
+        index: test-index
+        id: doc_1
+        body:
+          doc:
+            non_inference_field: "another non inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match:  { _source.sparse_field.text: "inference test" }
+  - match:  { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match:  { _source.sparse_field.inference.chunks.0.embeddings: $sparse_field_embedding }
+  - match:  { _source.dense_field.text: "another inference test" }
+  - match:  { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match:  { _source.dense_field.inference.chunks.0.embeddings: $dense_field_embedding }
+  - match:  { _source.non_inference_field: "another non inference test" }
+
+---
+"Updating semantic_text fields recalculates embeddings":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: "inference test"
+          dense_field: "another inference test"
+          non_inference_field: "non inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match:  { _source.sparse_field.text: "inference test" }
+  - match:  { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match:  { _source.dense_field.text: "another inference test" }
+  - match:  { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match:  { _source.non_inference_field: "non inference test" }
+
+  - do:
+      bulk:
+        index: test-index
+        body:
+          - '{"update": {"_id": "doc_1"}}'
+          - '{"doc":{"sparse_field": "I am a test", "dense_field": "I am a teapot"}}'
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match:  { _source.sparse_field.text: "I am a test" }
+  - match:  { _source.sparse_field.inference.chunks.0.text: "I am a test" }
+  - match:  { _source.dense_field.text: "I am a teapot" }
+  - match:  { _source.dense_field.inference.chunks.0.text: "I am a teapot" }
+  - match:  { _source.non_inference_field: "non inference test" }
+
+  - do:
+      update:
+        index: test-index
+        id: doc_1
+        body:
+          doc:
+            sparse_field: "updated inference test"
+            dense_field: "another updated inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match:  { _source.sparse_field.text: "updated inference test" }
+  - match:  { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
+  - match:  { _source.dense_field.text: "another updated inference test" }
+  - match:  { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
+  - match:  { _source.non_inference_field: "non inference test" }
+
+  - do:
+      bulk:
+        index: test-index
+        body:
+          - '{"update": {"_id": "doc_1"}}'
+          - '{"doc":{"sparse_field": "bulk inference test", "dense_field": "bulk updated inference test"}}'
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match:  { _source.sparse_field.text: "bulk inference test" }
+  - match:  { _source.sparse_field.inference.chunks.0.text: "bulk inference test" }
+  - match:  { _source.dense_field.text: "bulk updated inference test" }
+  - match:  { _source.dense_field.inference.chunks.0.text: "bulk updated inference test" }
+  - match:  { _source.non_inference_field: "non inference test" }
+
+---
+"Update logic handles source fields in object fields":
+  - do:
+      indices.create:
+        index: test-copy-to-index
+        body:
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              object_source:
+                properties:
+                  sparse_field:
+                    type: text
+                    copy_to: sparse_field
+                  dense_field:
+                    type: text
+                    copy_to: dense_field
+
+  - do:
+      index:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          sparse_field: "sparse data 1"
+          dense_field: "dense data 1"
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "sparse data 1" }
+  - length: { _source.sparse_field.inference.chunks: 1 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 1" }
+  - match: { _source.dense_field.text: "dense data 1" }
+  - length: { _source.dense_field.inference.chunks: 1 }
+  - match: { _source.dense_field.inference.chunks.0.text: "dense data 1" }
+
+  - do:
+      bulk:
+        index: test-copy-to-index
+        body:
+          - '{"update": {"_id": "doc_1"}}'
+          - >
+           {
+             "doc": {
+               "sparse_field": "sparse data 1",
+               "object_source.sparse_field": "sparse data 2",
+               "dense_field": "dense data 1",
+               "object_source.dense_field": "dense data 2"
+             }
+           }
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "sparse data 1" }
+  - length: { _source.sparse_field.inference.chunks: 2 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 2" }
+  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 1" }
+  - match: { _source.dense_field.text: "dense data 1" }
+  - length: { _source.dense_field.inference.chunks: 2 }
+  - match: { _source.dense_field.inference.chunks.0.text: "dense data 1" }
+  - match: { _source.dense_field.inference.chunks.1.text: "dense data 2" }
+
+  - do:
+      update:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          doc:
+            {
+              "sparse_field": "sparse data 1",
+              "object_source.sparse_field": "sparse data 3",
+              "dense_field": "dense data 1",
+              "object_source.dense_field": "dense data 3"
+            }
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "sparse data 1" }
+  - length: { _source.sparse_field.inference.chunks: 2 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 3" }
+  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 1" }
+  - match: { _source.dense_field.text: "dense data 1" }
+  - length: { _source.dense_field.inference.chunks: 2 }
+  - match: { _source.dense_field.inference.chunks.0.text: "dense data 1" }
+  - match: { _source.dense_field.inference.chunks.1.text: "dense data 3" }
+
+---
+"Updates fail when the updated value is invalid":
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: "inference test"
+          dense_field: "another inference test"
+          non_inference_field: "non inference test"
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.dense_field.text: "another inference test" }
+  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.non_inference_field: "non inference test" }
+
+  - do:
+      bulk:
+        index: test-index
+        body:
+          - '{"update": {"_id": "doc_1"}}'
+          - '{"doc":{"sparse_field": [{"key": "value"}], "dense_field": [{"key": "value"}]}}'
+
+  - match: { errors: true }
+  - match: { items.0.update.status: 400 }
+  - match: { items.0.update.error.reason: "/Invalid\\ format\\ for\\ field\\ \\[(dense|sparse)_field\\].+/" }
+
+  - do:
+      catch: bad_request
+      update:
+        index: test-index
+        id: doc_1
+        body:
+          doc: { "sparse_field": [{"key": "value"}], "dense_field": [{"key": "value"}] }
+
+  - match: { error.type: "status_exception" }
+  - match: { error.reason: "/Invalid\\ format\\ for\\ field\\ \\[(dense|sparse)_field\\].+/" }
+
+---
+"Partial updates work when using the update API":
+  - do:
+      indices.create:
+        index: test-copy-to-index
+        body:
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              sparse_source_field:
+                type: text
+                copy_to: sparse_field
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              dense_source_field:
+                type: text
+                copy_to: dense_field
+
+  - do:
+      index:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          sparse_field: "sparse data 1"
+          sparse_source_field: "sparse data 2"
+          dense_field: "dense data 1"
+          dense_source_field: "dense data 2"
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - length: { _source.sparse_field.inference.chunks: 2 }
+  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 2" }
+  - exists: _source.sparse_field.inference.chunks.1.embeddings
+  - length: { _source.dense_field.inference.chunks: 2 }
+  - match: { _source.dense_field.inference.chunks.1.text: "dense data 2" }
+  - exists: _source.dense_field.inference.chunks.1.embeddings
+
+  - do:
+      update:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          doc: { "sparse_source_field": "sparse data 3", "dense_source_field": "dense data 3" }
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "sparse data 1" }
+  - length: { _source.sparse_field.inference.chunks: 2 }
+  - match: { _source.sparse_field.inference.chunks.1.text: "sparse data 3" }
+  - exists: _source.sparse_field.inference.chunks.1.embeddings
+  - match: { _source.dense_field.text: "dense data 1" }
+  - length: { _source.dense_field.inference.chunks: 2 }
+  - match: { _source.dense_field.inference.chunks.1.text: "dense data 3" }
+  - exists: _source.dense_field.inference.chunks.1.embeddings
+
+---
+"Partial updates work when using the update API and the semantic_text field's original value is null":
+  - do:
+      indices.create:
+        index: test-copy-to-index
+        body:
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              sparse_source_field:
+                type: text
+                copy_to: sparse_field
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+              dense_source_field:
+                type: text
+                copy_to: dense_field
+
+  # Don't set sparse_field or dense_field so their original value is null
+  - do:
+      index:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          sparse_source_field: "sparse data 2"
+          dense_source_field: "dense data 2"
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: null }
+  - length: { _source.sparse_field.inference.chunks: 1 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 2" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.text: null }
+  - length: { _source.dense_field.inference.chunks: 1 }
+  - match: { _source.dense_field.inference.chunks.0.text: "dense data 2" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+
+  - do:
+      update:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          doc: { "sparse_source_field": "sparse data 3", "dense_source_field": "dense data 3" }
+
+  - do:
+      get:
+        index: test-copy-to-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: null }
+  - length: { _source.sparse_field.inference.chunks: 1 }
+  - match: { _source.sparse_field.inference.chunks.0.text: "sparse data 3" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.text: null }
+  - length: { _source.dense_field.inference.chunks: 1 }
+  - match: { _source.dense_field.inference.chunks.0.text: "dense data 3" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+
+---
+"Updates with script are not allowed":
+  - do:
+      bulk:
+        index: test-index
+        body:
+          - '{"index": {"_id": "doc_1"}}'
+          - '{"doc":{"sparse_field": "I am a test", "dense_field": "I am a teapot"}}'
+
+  - do:
+      bulk:
+        index: test-index
+        body:
+          - '{"update": {"_id": "doc_1"}}'
+          - '{"script": "ctx._source.new_field = \"hello\"", "scripted_upsert": true}'
+
+  - match: { errors: true }
+  - match: { items.0.update.status: 400 }
+  - match: { items.0.update.error.reason: "Cannot apply update with a script on indices that contain [semantic_text] field(s)" }
+
+  - do:
+      catch: bad_request
+      update:
+        index: test-index
+        id: doc_1
+        body:
+          script:
+            source: "ctx._source.new_field = \"hello\""
+            lang: "painless"
+
+  - match: { error.type: "status_exception" }
+  - match: { error.reason: "Cannot apply update with a script on indices that contain inference field(s)" }
+
+---
+"semantic_text copy_to needs values for every source field for bulk updates":
+  - do:
+      indices.create:
+        index: test-copy-to-index
+        body:
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+              source_field:
+                type: text
+                copy_to: sparse_field
+              another_source_field:
+                type: text
+                copy_to: sparse_field
+
+  # Not every source field needed on creation
+  - do:
+      index:
+        index: test-copy-to-index
+        id: doc_1
+        body:
+          source_field: "a single source field provided"
+          sparse_field: "inference test"
+
+  # Every source field needed on bulk updates
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-copy-to-index", "_id": "doc_1"}}'
+          - '{"doc": {"source_field": "a single source field is kept as provided via bulk", "sparse_field": "updated inference test" }}'
+
+  - match: { items.0.update.status: 400 }
+  - match: { items.0.update.error.reason: "Field [another_source_field] must be specified on an update request to calculate inference for field [sparse_field]" }
+
+---
+"Calculates embeddings for bulk operations - update":
+  - do:
+      bulk:
+        body:
+          - '{"index": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test"}'
+
+  - match: { errors: false }
+  - match: { items.0.index.result: "created" }
+
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": "updated inference test", "dense_field": "another updated inference test", "non_inference_field": "updated non inference test" }}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "updated" }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "updated inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
+  - match: { _source.dense_field.text: "another updated inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
+  - match: { _source.non_inference_field: "updated non inference test" }
+
+  # Script update not supported
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"script": {"source": {"ctx.sparse_field": "updated inference test"}}}'
+
+  - match: { errors: true }
+  - match: { items.0.update.status: 400 }
+  - match: { items.0.update.error.reason: "Cannot apply update with a script on indices that contain [semantic_text] field(s)" }
+
+---
+"Calculates embeddings for bulk operations - upsert":
+  # Initial update fails
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }}'
+
+  - match: { errors: true }
+  - match: { items.0.update.status: 404 }
+
+  # Update as upsert
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }, "doc_as_upsert": true}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "created" }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "inference test" }
+  - match: { _source.dense_field.text: "another inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another inference test" }
+  - match: { _source.non_inference_field: "non inference test" }
+
+  - do:
+      bulk:
+        body:
+          - '{"update": {"_index": "test-index", "_id": "doc_1"}}'
+          - '{"doc": { "sparse_field": "updated inference test", "dense_field": "another updated inference test", "non_inference_field": "updated non inference test" }, "doc_as_upsert": true}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: "updated" }
+
+  - do:
+      get:
+        index: test-index
+        id: doc_1
+
+  - match: { _source.sparse_field.text: "updated inference test" }
+  - exists: _source.sparse_field.inference.chunks.0.embeddings
+  - match: { _source.sparse_field.inference.chunks.0.text: "updated inference test" }
+  - match: { _source.dense_field.text: "another updated inference test" }
+  - exists: _source.dense_field.inference.chunks.0.embeddings
+  - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" }
+  - match: { _source.non_inference_field: "updated non inference test" }

From f73fedc52ad87d961213045945f0d3f9383f3124 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Tue, 28 May 2024 21:40:00 +0100
Subject: [PATCH 019/208] Fix trappy timeouts in ILM (#109112)

Relates #107984
---
 .../existence/FrozenExistenceDeciderIT.java   |  2 +-
 .../xpack/core/ilm/StartILMRequest.java       |  6 +-
 .../xpack/core/ilm/StopILMRequest.java        |  6 +-
 .../ilm/action/DeleteLifecycleAction.java     | 14 ++--
 .../core/ilm/action/GetLifecycleAction.java   | 10 +--
 .../core/ilm/action/PutLifecycleRequest.java  | 32 +++++----
 .../RemoveIndexLifecyclePolicyAction.java     |  9 +--
 .../core/template/IndexTemplateRegistry.java  |  4 +-
 .../xpack/core/ilm/StartILMRequestTests.java  |  2 +-
 .../xpack/core/ilm/StopILMRequestTests.java   |  2 +-
 .../action/DeleteLifecycleRequestTests.java   |  4 +-
 .../ilm/action/GetLifecycleRequestTests.java  |  4 +-
 .../ilm/action/PutLifecycleRequestTests.java  | 20 +++++-
 ...emoveIndexLifecyclePolicyRequestTests.java |  9 ++-
 .../downsample/ILMDownsampleDisruptionIT.java |  2 +-
 .../ClusterStateWaitThresholdBreachTests.java |  2 +-
 ...ataStreamAndIndexLifecycleMixingTests.java | 10 +--
 .../xpack/ilm/DataTiersMigrationsTests.java   |  4 +-
 .../xpack/ilm/ILMMultiNodeIT.java             |  2 +-
 .../ilm/ILMMultiNodeWithCCRDisabledIT.java    |  2 +-
 .../IndexLifecycleInitialisationTests.java    | 65 ++++++++++++-------
 .../ilm/action/ReservedLifecycleAction.java   | 10 ++-
 .../ilm/action/RestDeleteLifecycleAction.java |  8 ++-
 .../ilm/action/RestGetLifecycleAction.java    |  8 ++-
 .../ilm/action/RestPutLifecycleAction.java    | 23 ++++---
 .../RestRemoveIndexLifecyclePolicyAction.java |  8 ++-
 .../xpack/ilm/action/RestStartILMAction.java  |  4 +-
 .../xpack/ilm/action/RestStopAction.java      |  4 +-
 .../TransportDeleteLifecycleAction.java       |  9 ---
 .../ilm/LifecycleOperationSnapshotTests.java  |  4 +-
 .../TransportDeleteLifecycleActionTests.java  |  6 +-
 .../TransportPutLifecycleActionTests.java     | 12 +++-
 .../action/TransportStopILMActionTests.java   |  2 +-
 33 files changed, 183 insertions(+), 126 deletions(-)

diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java
index c72d5e83d2bd3..3e729e9139c2b 100644
--- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java
+++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java
@@ -99,7 +99,7 @@ public void testZeroToOne() throws Exception {
             singletonMap(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(fsRepoName, randomBoolean()))
         );
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy("policy", Map.of("hot", hotPhase, "frozen", frozenPhase));
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
         Settings settings = Settings.builder()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/StartILMRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/StartILMRequest.java
index c790d76746c1d..214803d2d7679 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/StartILMRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/StartILMRequest.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -16,11 +17,10 @@ public class StartILMRequest extends AcknowledgedRequest<StartILMRequest> {
 
     public StartILMRequest(StreamInput in) throws IOException {
         super(in);
-
     }
 
-    public StartILMRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+    public StartILMRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+        super(masterNodeTimeout, ackTimeout);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/StopILMRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/StopILMRequest.java
index 82c6d8ba4800b..130795dd69d11 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/StopILMRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/StopILMRequest.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -16,11 +17,10 @@ public class StopILMRequest extends AcknowledgedRequest<StopILMRequest> {
 
     public StopILMRequest(StreamInput in) throws IOException {
         super(in);
-
     }
 
-    public StopILMRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+    public StopILMRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+        super(masterNodeTimeout, ackTimeout);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/DeleteLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/DeleteLifecycleAction.java
index 01adb18caac7d..3a97954b1f2dd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/DeleteLifecycleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/DeleteLifecycleAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -29,12 +29,10 @@ protected DeleteLifecycleAction() {
 
     public static class Request extends AcknowledgedRequest<Request> {
 
-        public static final ParseField POLICY_FIELD = new ParseField("policy");
+        private final String policyName;
 
-        private String policyName;
-
-        public Request(String policyName) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String policyName) {
+            super(masterNodeTimeout, ackTimeout);
             this.policyName = policyName;
         }
 
@@ -43,10 +41,6 @@ public Request(StreamInput in) throws IOException {
             policyName = in.readString();
         }
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-        }
-
         public String getPolicyName() {
             return policyName;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleAction.java
index d359498f33621..60042ab666c60 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -103,8 +104,8 @@ public Iterator<ToXContent> toXContentChunked(ToXContent.Params outerParams) {
     public static class Request extends AcknowledgedRequest<Request> {
         private final String[] policyNames;
 
-        public Request(String... policyNames) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String... policyNames) {
+            super(masterNodeTimeout, ackTimeout);
             if (policyNames == null) {
                 throw new IllegalArgumentException("ids cannot be null");
             }
@@ -116,11 +117,6 @@ public Request(StreamInput in) throws IOException {
             policyNames = in.readStringArray();
         }
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-            policyNames = Strings.EMPTY_ARRAY;
-        }
-
         @Override
         public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
             return new CancellableTask(id, type, action, "get-lifecycle-task", parentTaskId, headers);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequest.java
index ebaaf42246251..c955c30c163eb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequest.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -25,20 +26,31 @@
 
 public class PutLifecycleRequest extends AcknowledgedRequest<PutLifecycleRequest> implements ToXContentObject {
 
+    public interface Factory {
+        PutLifecycleRequest create(LifecyclePolicy lifecyclePolicy);
+
+        String getPolicyName();
+    }
+
     public static final ParseField POLICY_FIELD = new ParseField("policy");
-    private static final ConstructingObjectParser<PutLifecycleRequest, String> PARSER = new ConstructingObjectParser<>(
+    private static final ConstructingObjectParser<PutLifecycleRequest, Factory> PARSER = new ConstructingObjectParser<>(
         "put_lifecycle_request",
-        a -> new PutLifecycleRequest((LifecyclePolicy) a[0])
+        false,
+        (a, factory) -> factory.create((LifecyclePolicy) a[0])
     );
 
     static {
-        PARSER.declareObject(ConstructingObjectParser.constructorArg(), LifecyclePolicy::parse, POLICY_FIELD);
+        PARSER.declareObject(
+            ConstructingObjectParser.constructorArg(),
+            (parser, factory) -> LifecyclePolicy.parse(parser, factory.getPolicyName()),
+            POLICY_FIELD
+        );
     }
 
-    private LifecyclePolicy policy;
+    private final LifecyclePolicy policy;
 
-    public PutLifecycleRequest(LifecyclePolicy policy) {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+    public PutLifecycleRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout, LifecyclePolicy policy) {
+        super(masterNodeTimeout, ackTimeout);
         this.policy = policy;
     }
 
@@ -47,10 +59,6 @@ public PutLifecycleRequest(StreamInput in) throws IOException {
         policy = new LifecyclePolicy(in);
     }
 
-    public PutLifecycleRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-    }
-
     public LifecyclePolicy getPolicy() {
         return policy;
     }
@@ -71,8 +79,8 @@ public ActionRequestValidationException validate() {
         return err;
     }
 
-    public static PutLifecycleRequest parseRequest(String name, XContentParser parser) {
-        return PARSER.apply(parser, name);
+    public static PutLifecycleRequest parseRequest(Factory factory, XContentParser parser) {
+        return PARSER.apply(parser, factory);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyAction.java
index e45e4096e0a81..21ce966d17828 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -113,12 +114,8 @@ public Request(StreamInput in) throws IOException {
             indicesOptions = IndicesOptions.readIndicesOptions(in);
         }
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-        }
-
-        public Request(String... indices) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String... indices) {
+            super(masterNodeTimeout, ackTimeout);
             if (indices == null) {
                 throw new IllegalArgumentException("indices cannot be null");
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java
index e189116b0179c..87092c45bf032 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java
@@ -76,6 +76,8 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener {
 
     private static final Logger logger = LogManager.getLogger(IndexTemplateRegistry.class);
 
+    private static final TimeValue REGISTRY_ACTION_TIMEOUT = TimeValue.THIRTY_SECONDS; // TODO should this be longer?
+
     protected final Settings settings;
     protected final Client client;
     protected final ThreadPool threadPool;
@@ -614,7 +616,7 @@ protected boolean isUpgradeRequired(LifecyclePolicy currentPolicy, LifecyclePoli
     private void putPolicy(final LifecyclePolicy policy, final AtomicBoolean creationCheck) {
         final Executor executor = threadPool.generic();
         executor.execute(() -> {
-            PutLifecycleRequest request = new PutLifecycleRequest(policy);
+            PutLifecycleRequest request = new PutLifecycleRequest(REGISTRY_ACTION_TIMEOUT, REGISTRY_ACTION_TIMEOUT, policy);
             request.masterNodeTimeout(TimeValue.MAX_VALUE);
             executeAsyncWithOrigin(
                 client.threadPool().getThreadContext(),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StartILMRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StartILMRequestTests.java
index 308cb1a948916..f2252e53724aa 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StartILMRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StartILMRequestTests.java
@@ -14,7 +14,7 @@ public class StartILMRequestTests extends AbstractWireSerializingTestCase<StartI
 
     @Override
     protected StartILMRequest createTestInstance() {
-        return new StartILMRequest();
+        return new StartILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StopILMRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StopILMRequestTests.java
index cb40127272b0e..9c97e8422fed8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StopILMRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StopILMRequestTests.java
@@ -14,7 +14,7 @@ public class StopILMRequestTests extends AbstractWireSerializingTestCase<StopILM
 
     @Override
     protected StopILMRequest createTestInstance() {
-        return new StopILMRequest();
+        return new StopILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/DeleteLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/DeleteLifecycleRequestTests.java
index ddca47f721b9d..b7cf8f4dcf218 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/DeleteLifecycleRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/DeleteLifecycleRequestTests.java
@@ -14,7 +14,7 @@ public class DeleteLifecycleRequestTests extends AbstractWireSerializingTestCase
 
     @Override
     protected Request createTestInstance() {
-        return new Request(randomAlphaOfLengthBetween(1, 20));
+        return new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomAlphaOfLengthBetween(1, 20));
     }
 
     @Override
@@ -24,7 +24,7 @@ protected Writeable.Reader<Request> instanceReader() {
 
     @Override
     protected Request mutateInstance(Request request) {
-        return new Request(request.getPolicyName() + randomAlphaOfLengthBetween(1, 10));
+        return new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, request.getPolicyName() + randomAlphaOfLengthBetween(1, 10));
     }
 
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleRequestTests.java
index 6aa55eb7f7a0a..fc963a7bc45e2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/GetLifecycleRequestTests.java
@@ -16,7 +16,7 @@ public class GetLifecycleRequestTests extends AbstractWireSerializingTestCase<Re
 
     @Override
     protected Request createTestInstance() {
-        return new Request(randomAlphaOfLengthBetween(1, 20));
+        return new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomAlphaOfLengthBetween(1, 20));
     }
 
     @Override
@@ -29,6 +29,6 @@ protected Request mutateInstance(Request request) {
         String[] originalPolicies = request.getPolicyNames();
         String[] newPolicies = Arrays.copyOf(originalPolicies, originalPolicies.length + 1);
         newPolicies[originalPolicies.length] = randomAlphaOfLength(5);
-        return new Request(newPolicies);
+        return new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, newPolicies);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java
index ec821aa588c7b..feb5ca24a021d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/PutLifecycleRequestTests.java
@@ -48,7 +48,11 @@ public void setup() {
 
     @Override
     protected PutLifecycleRequest createTestInstance() {
-        return new PutLifecycleRequest(LifecyclePolicyTests.randomTimeseriesLifecyclePolicy(lifecycleName));
+        return new PutLifecycleRequest(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            LifecyclePolicyTests.randomTimeseriesLifecyclePolicy(lifecycleName)
+        );
     }
 
     @Override
@@ -58,7 +62,17 @@ protected Writeable.Reader<PutLifecycleRequest> instanceReader() {
 
     @Override
     protected PutLifecycleRequest doParseInstance(XContentParser parser) {
-        return PutLifecycleRequest.parseRequest(lifecycleName, parser);
+        return PutLifecycleRequest.parseRequest(new PutLifecycleRequest.Factory() {
+            @Override
+            public PutLifecycleRequest create(LifecyclePolicy lifecyclePolicy) {
+                return new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
+            }
+
+            @Override
+            public String getPolicyName() {
+                return lifecycleName;
+            }
+        }, parser);
     }
 
     @Override
@@ -130,7 +144,7 @@ protected PutLifecycleRequest mutateInstance(PutLifecycleRequest request) {
             request.getPolicy(),
             () -> LifecyclePolicyTests.randomTimeseriesLifecyclePolicy(name)
         );
-        return new PutLifecycleRequest(policy);
+        return new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policy);
     }
 
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyRequestTests.java
index 4bb58540a14e2..f998e15d0b798 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyRequestTests.java
@@ -18,7 +18,7 @@ public class RemoveIndexLifecyclePolicyRequestTests extends AbstractWireSerializ
 
     @Override
     protected Request createTestInstance() {
-        Request request = new Request(generateRandomStringArray(20, 20, false));
+        Request request = new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, generateRandomStringArray(20, 20, false));
         if (randomBoolean()) {
             IndicesOptions indicesOptions = IndicesOptions.fromOptions(
                 randomBoolean(),
@@ -67,13 +67,16 @@ protected Request mutateInstance(Request instance) {
             );
             default -> throw new AssertionError("Illegal randomisation branch");
         }
-        Request newRequest = new Request(indices);
+        Request newRequest = new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indices);
         newRequest.indicesOptions(indicesOptions);
         return newRequest;
     }
 
     public void testNullIndices() {
-        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new Request((String[]) null));
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, (String[]) null)
+        );
         assertEquals("indices cannot be null", exception.getMessage());
     }
 
diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java
index e4091af0fedf2..8a0d9edae4993 100644
--- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java
+++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java
@@ -136,7 +136,7 @@ public void setup(final String sourceIndex, int numOfShards, int numOfReplicas,
             )
         );
         LifecyclePolicy policy = new LifecyclePolicy(POLICY_NAME, phases);
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(policy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).actionGet());
     }
 
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java
index 7602a2cd16e78..55daa8104c12a 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java
@@ -91,7 +91,7 @@ public void testWaitInShrunkShardsAllocatedExceedsThreshold() throws Exception {
             Map.of(MigrateAction.NAME, MigrateAction.DISABLED, ShrinkAction.NAME, new ShrinkAction(1, null, false))
         );
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, Map.of("warm", warmPhase));
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
         // we're configuring a very high number of replicas. this will make ths shrunk index unable to allocate successfully, so ILM will
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java
index 7278b0e6c7f49..6d6f292d112f9 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java
@@ -107,7 +107,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception {
         RolloverAction rolloverIlmAction = new RolloverAction(RolloverConditions.newBuilder().addMaxIndexDocsCondition(2L).build());
         Phase hotPhase = new Phase("hot", TimeValue.ZERO, Map.of(rolloverIlmAction.getWriteableName(), rolloverIlmAction));
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, Map.of("hot", hotPhase));
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
         putComposableIndexTemplate(
@@ -289,7 +289,7 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro
         RolloverAction rolloverIlmAction = new RolloverAction(RolloverConditions.newBuilder().addMaxIndexDocsCondition(2L).build());
         Phase hotPhase = new Phase("hot", TimeValue.ZERO, Map.of(rolloverIlmAction.getWriteableName(), rolloverIlmAction));
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, Map.of("hot", hotPhase));
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
         putComposableIndexTemplate(
@@ -464,7 +464,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce
         RolloverAction rolloverIlmAction = new RolloverAction(RolloverConditions.newBuilder().addMaxIndexDocsCondition(2L).build());
         Phase hotPhase = new Phase("hot", TimeValue.ZERO, Map.of(rolloverIlmAction.getWriteableName(), rolloverIlmAction));
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, Map.of("hot", hotPhase));
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
         putComposableIndexTemplate(
@@ -718,7 +718,7 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw
         RolloverAction rolloverIlmAction = new RolloverAction(RolloverConditions.newBuilder().addMaxIndexDocsCondition(2L).build());
         Phase hotPhase = new Phase("hot", TimeValue.ZERO, Map.of(rolloverIlmAction.getWriteableName(), rolloverIlmAction));
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, Map.of("hot", hotPhase));
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
         // let's update the index template to remove the data stream lifecycle configuration and replace it with an ILM configuration
@@ -813,7 +813,7 @@ public void testGetDataStreamResponse() throws Exception {
         RolloverAction rolloverIlmAction = new RolloverAction(RolloverConditions.newBuilder().addMaxIndexDocsCondition(2L).build());
         Phase hotPhase = new Phase("hot", TimeValue.ZERO, Map.of(rolloverIlmAction.getWriteableName(), rolloverIlmAction));
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, Map.of("hot", hotPhase));
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
         putComposableIndexTemplate(
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java
index bf5ab23823614..01288b4135d15 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java
@@ -105,7 +105,7 @@ public void testIndexDataTierMigration() throws Exception {
         Phase warmPhase = new Phase("warm", TimeValue.ZERO, Collections.emptyMap());
         Phase coldPhase = new Phase("cold", TimeValue.ZERO, Collections.emptyMap());
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, Map.of("hot", hotPhase, "warm", warmPhase, "cold", coldPhase));
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
         Settings settings = Settings.builder()
@@ -166,7 +166,7 @@ public void testUserOptsOutOfTierMigration() throws Exception {
         Phase warmPhase = new Phase("warm", TimeValue.ZERO, Collections.emptyMap());
         Phase coldPhase = new Phase("cold", TimeValue.ZERO, Collections.emptyMap());
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, Map.of("hot", hotPhase, "warm", warmPhase, "cold", coldPhase));
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
         Settings settings = Settings.builder()
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java
index 46c47869d8651..b443c769407c5 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java
@@ -76,7 +76,7 @@ public void testShrinkOnTiers() throws Exception {
         phases.put(hotPhase.getName(), hotPhase);
         phases.put(warmPhase.getName(), warmPhase);
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy("shrink-policy", phases);
-        client().execute(ILMActions.PUT, new PutLifecycleRequest(lifecyclePolicy)).get();
+        client().execute(ILMActions.PUT, new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy)).get();
 
         Template t = new Template(
             Settings.builder()
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java
index 07c82f3dcfe98..e02dd5fe45676 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java
@@ -76,7 +76,7 @@ public void testShrinkOnTiers() throws Exception {
         Phase hotPhase = new Phase("hot", TimeValue.ZERO, actions);
 
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy("shrink-policy", Collections.singletonMap(hotPhase.getName(), hotPhase));
-        client().execute(ILMActions.PUT, new PutLifecycleRequest(lifecyclePolicy)).get();
+        client().execute(ILMActions.PUT, new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy)).get();
 
         Template t = new Template(
             Settings.builder()
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
index 209330a3be2de..3530f33704beb 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
@@ -144,24 +144,29 @@ public void testSingleNodeCluster() throws Exception {
         // test get-lifecycle behavior when IndexLifecycleMetadata is null
         GetLifecycleAction.Response getUninitializedLifecycleResponse = client().execute(
             GetLifecycleAction.INSTANCE,
-            new GetLifecycleAction.Request()
+            new GetLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
         ).get();
         assertThat(getUninitializedLifecycleResponse.getPolicies().size(), equalTo(0));
         ExecutionException exception = expectThrows(
             ExecutionException.class,
-            () -> client().execute(GetLifecycleAction.INSTANCE, new GetLifecycleAction.Request("non-existent-policy")).get()
+            () -> client().execute(
+                GetLifecycleAction.INSTANCE,
+                new GetLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "non-existent-policy")
+            ).get()
         );
         assertThat(exception.getMessage(), containsString("Lifecycle policy not found: [non-existent-policy]"));
 
         logger.info("Creating lifecycle [test_lifecycle]");
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         long lowerBoundModifiedDate = Instant.now().toEpochMilli();
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
         long upperBoundModifiedDate = Instant.now().toEpochMilli();
 
         // assert version and modified_date
-        GetLifecycleAction.Response getLifecycleResponse = client().execute(GetLifecycleAction.INSTANCE, new GetLifecycleAction.Request())
-            .get();
+        GetLifecycleAction.Response getLifecycleResponse = client().execute(
+            GetLifecycleAction.INSTANCE,
+            new GetLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+        ).get();
         assertThat(getLifecycleResponse.getPolicies().size(), equalTo(1));
         GetLifecycleAction.LifecyclePolicyResponseItem responseItem = getLifecycleResponse.getPolicies().get(0);
         assertThat(responseItem.getLifecyclePolicy(), equalTo(lifecyclePolicy));
@@ -199,21 +204,26 @@ public void testNoOpPolicyUpdates() throws Exception {
         phases.put("hot", new Phase("hot", TimeValue.ZERO, Map.of()));
         LifecyclePolicy policy = new LifecyclePolicy("mypolicy", phases);
 
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(policy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
-        GetLifecycleAction.Response getLifecycleResponse = client().execute(GetLifecycleAction.INSTANCE, new GetLifecycleAction.Request())
-            .get();
+        GetLifecycleAction.Response getLifecycleResponse = client().execute(
+            GetLifecycleAction.INSTANCE,
+            new GetLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+        ).get();
         assertThat(getLifecycleResponse.getPolicies().size(), equalTo(1));
         GetLifecycleAction.LifecyclePolicyResponseItem responseItem = getLifecycleResponse.getPolicies().get(0);
         assertThat(responseItem.getLifecyclePolicy(), equalTo(policy));
         assertThat(responseItem.getVersion(), equalTo(1L));
 
         // Put the same policy in place, which should be a no-op
-        putLifecycleRequest = new PutLifecycleRequest(policy);
+        putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
-        getLifecycleResponse = client().execute(GetLifecycleAction.INSTANCE, new GetLifecycleAction.Request()).get();
+        getLifecycleResponse = client().execute(
+            GetLifecycleAction.INSTANCE,
+            new GetLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+        ).get();
         assertThat(getLifecycleResponse.getPolicies().size(), equalTo(1));
         responseItem = getLifecycleResponse.getPolicies().get(0);
         assertThat(responseItem.getLifecyclePolicy(), equalTo(policy));
@@ -225,10 +235,13 @@ public void testNoOpPolicyUpdates() throws Exception {
         newPhases.put("cold", new Phase("cold", TimeValue.timeValueDays(1), Map.of()));
         policy = new LifecyclePolicy("mypolicy", newPhases);
 
-        putLifecycleRequest = new PutLifecycleRequest(policy);
+        putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
-        getLifecycleResponse = client().execute(GetLifecycleAction.INSTANCE, new GetLifecycleAction.Request()).get();
+        getLifecycleResponse = client().execute(
+            GetLifecycleAction.INSTANCE,
+            new GetLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+        ).get();
         assertThat(getLifecycleResponse.getPolicies().size(), equalTo(1));
         responseItem = getLifecycleResponse.getPolicies().get(0);
         assertThat(responseItem.getLifecyclePolicy(), equalTo(policy));
@@ -241,11 +254,13 @@ public void testExplainExecution() throws Exception {
         logger.info("Starting server1");
         internalCluster().startNode();
         logger.info("Creating lifecycle [test_lifecycle]");
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
-        GetLifecycleAction.Response getLifecycleResponse = client().execute(GetLifecycleAction.INSTANCE, new GetLifecycleAction.Request())
-            .get();
+        GetLifecycleAction.Response getLifecycleResponse = client().execute(
+            GetLifecycleAction.INSTANCE,
+            new GetLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+        ).get();
         assertThat(getLifecycleResponse.getPolicies().size(), equalTo(1));
         GetLifecycleAction.LifecyclePolicyResponseItem responseItem = getLifecycleResponse.getPolicies().get(0);
         assertThat(responseItem.getLifecyclePolicy(), equalTo(lifecyclePolicy));
@@ -314,11 +329,13 @@ public void testExplainParseOriginationDate() throws Exception {
         logger.info("Starting server2");
         internalCluster().startNode();
         logger.info("Creating lifecycle [test_lifecycle]");
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
 
-        GetLifecycleAction.Response getLifecycleResponse = client().execute(GetLifecycleAction.INSTANCE, new GetLifecycleAction.Request())
-            .get();
+        GetLifecycleAction.Response getLifecycleResponse = client().execute(
+            GetLifecycleAction.INSTANCE,
+            new GetLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+        ).get();
         assertThat(getLifecycleResponse.getPolicies().size(), equalTo(1));
         GetLifecycleAction.LifecyclePolicyResponseItem responseItem = getLifecycleResponse.getPolicies().get(0);
         assertThat(responseItem.getLifecyclePolicy(), equalTo(lifecyclePolicy));
@@ -402,7 +419,7 @@ public void testMasterDedicatedDataDedicated() throws Exception {
         }
 
         logger.info("Creating lifecycle [test_lifecycle]");
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
         logger.info("Creating index [test]");
         CreateIndexResponse createIndexResponse = indicesAdmin().create(new CreateIndexRequest("test").settings(settings)).actionGet();
@@ -430,7 +447,7 @@ public void testCreatePolicyWhenStopped() throws Exception {
         final String server_1 = internalCluster().startNode();
         final String node1 = getLocalNodeId(server_1);
 
-        assertAcked(client().execute(ILMActions.STOP, new StopILMRequest()).get());
+        assertAcked(client().execute(ILMActions.STOP, new StopILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)).get());
         assertBusy(() -> {
             OperationMode mode = client().execute(
                 GetStatusAction.INSTANCE,
@@ -441,14 +458,16 @@ public void testCreatePolicyWhenStopped() throws Exception {
         });
 
         logger.info("Creating lifecycle [test_lifecycle]");
-        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(lifecyclePolicy);
+        PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
         long lowerBoundModifiedDate = Instant.now().toEpochMilli();
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).get());
         long upperBoundModifiedDate = Instant.now().toEpochMilli();
 
         // assert version and modified_date
-        GetLifecycleAction.Response getLifecycleResponse = client().execute(GetLifecycleAction.INSTANCE, new GetLifecycleAction.Request())
-            .get();
+        GetLifecycleAction.Response getLifecycleResponse = client().execute(
+            GetLifecycleAction.INSTANCE,
+            new GetLifecycleAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+        ).get();
         assertThat(getLifecycleResponse.getPolicies().size(), equalTo(1));
         GetLifecycleAction.LifecyclePolicyResponseItem responseItem = getLifecycleResponse.getPolicies().get(0);
         assertThat(responseItem.getLifecyclePolicy(), equalTo(lifecyclePolicy));
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleAction.java
index 947a028e9262e..ba41e29531b76 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleAction.java
@@ -7,8 +7,10 @@
 
 package org.elasticsearch.xpack.ilm.action;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
@@ -16,6 +18,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
+import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction;
 import org.elasticsearch.xpack.core.ilm.action.PutLifecycleRequest;
 
 import java.io.IOException;
@@ -61,7 +64,8 @@ public Collection<PutLifecycleRequest> prepare(Object input) throws IOException
         List<LifecyclePolicy> policies = (List<LifecyclePolicy>) input;
 
         for (var policy : policies) {
-            PutLifecycleRequest request = new PutLifecycleRequest(policy);
+            // timeouts don't matter here
+            PutLifecycleRequest request = new PutLifecycleRequest(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, policy);
             validate(request);
             result.add(request);
         }
@@ -93,7 +97,9 @@ public TransformState transform(Object source, TransformState prevState) throws
 
         for (var policyToDelete : toDelete) {
             TransportDeleteLifecycleAction.DeleteLifecyclePolicyTask task = new TransportDeleteLifecycleAction.DeleteLifecyclePolicyTask(
-                policyToDelete
+                // timeouts don't matter here
+                new DeleteLifecycleAction.Request(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS, policyToDelete),
+                ActionListener.noop()
             );
             state = task.execute(state);
         }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestDeleteLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestDeleteLifecycleAction.java
index 8e8ec714f380d..688925cd41fec 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestDeleteLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestDeleteLifecycleAction.java
@@ -34,9 +34,11 @@ public String getName() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
         String lifecycleName = restRequest.param("name");
-        DeleteLifecycleAction.Request deleteLifecycleRequest = new DeleteLifecycleAction.Request(lifecycleName);
-        deleteLifecycleRequest.ackTimeout(getAckTimeout(restRequest));
-        deleteLifecycleRequest.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        DeleteLifecycleAction.Request deleteLifecycleRequest = new DeleteLifecycleAction.Request(
+            getMasterNodeTimeout(restRequest),
+            getAckTimeout(restRequest),
+            lifecycleName
+        );
 
         return channel -> client.execute(DeleteLifecycleAction.INSTANCE, deleteLifecycleRequest, new RestToXContentListener<>(channel));
     }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetLifecycleAction.java
index 5a9923c014d6d..3de071a977dc6 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestGetLifecycleAction.java
@@ -36,9 +36,11 @@ public String getName() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
         String[] lifecycleNames = Strings.splitStringByCommaToArray(restRequest.param("name"));
-        GetLifecycleAction.Request getLifecycleRequest = new GetLifecycleAction.Request(lifecycleNames);
-        getLifecycleRequest.ackTimeout(getAckTimeout(restRequest));
-        getLifecycleRequest.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        GetLifecycleAction.Request getLifecycleRequest = new GetLifecycleAction.Request(
+            getMasterNodeTimeout(restRequest),
+            getAckTimeout(restRequest),
+            lifecycleNames
+        );
 
         return channel -> new RestCancellableNodeClient(client, restRequest.getHttpChannel()).execute(
             GetLifecycleAction.INSTANCE,
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestPutLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestPutLifecycleAction.java
index d07c498e21ec2..ba9dc4340474f 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestPutLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestPutLifecycleAction.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
 import org.elasticsearch.xpack.core.ilm.action.ILMActions;
 import org.elasticsearch.xpack.core.ilm.action.PutLifecycleRequest;
 
@@ -36,13 +36,20 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
-        String lifecycleName = restRequest.param("name");
-        try (XContentParser parser = restRequest.contentParser()) {
-            PutLifecycleRequest putLifecycleRequest = PutLifecycleRequest.parseRequest(lifecycleName, parser);
-            putLifecycleRequest.ackTimeout(getAckTimeout(restRequest));
-            putLifecycleRequest.masterNodeTimeout(getMasterNodeTimeout(restRequest));
-
-            return channel -> client.execute(ILMActions.PUT, putLifecycleRequest, new RestToXContentListener<>(channel));
+        final PutLifecycleRequest putLifecycleRequest;
+        try (var parser = restRequest.contentParser()) {
+            putLifecycleRequest = PutLifecycleRequest.parseRequest(new PutLifecycleRequest.Factory() {
+                @Override
+                public PutLifecycleRequest create(LifecyclePolicy lifecyclePolicy) {
+                    return new PutLifecycleRequest(getMasterNodeTimeout(restRequest), getAckTimeout(restRequest), lifecyclePolicy);
+                }
+
+                @Override
+                public String getPolicyName() {
+                    return restRequest.param("name");
+                }
+            }, parser);
         }
+        return channel -> client.execute(ILMActions.PUT, putLifecycleRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestRemoveIndexLifecyclePolicyAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestRemoveIndexLifecyclePolicyAction.java
index a011aa3d38b64..d750dfb257834 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestRemoveIndexLifecyclePolicyAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestRemoveIndexLifecyclePolicyAction.java
@@ -18,6 +18,7 @@
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 public class RestRemoveIndexLifecyclePolicyAction extends BaseRestHandler {
@@ -35,8 +36,11 @@ public String getName() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
         String[] indexes = Strings.splitStringByCommaToArray(restRequest.param("index"));
-        RemoveIndexLifecyclePolicyAction.Request changePolicyRequest = new RemoveIndexLifecyclePolicyAction.Request(indexes);
-        changePolicyRequest.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        RemoveIndexLifecyclePolicyAction.Request changePolicyRequest = new RemoveIndexLifecyclePolicyAction.Request(
+            getMasterNodeTimeout(restRequest),
+            getAckTimeout(restRequest),
+            indexes
+        );
         changePolicyRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, changePolicyRequest.indicesOptions()));
 
         return channel -> client.execute(
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestStartILMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestStartILMAction.java
index bba6261b323f9..ad0d9b0d289d0 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestStartILMAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestStartILMAction.java
@@ -34,9 +34,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        StartILMRequest request = new StartILMRequest();
-        request.ackTimeout(getAckTimeout(restRequest));
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        StartILMRequest request = new StartILMRequest(getMasterNodeTimeout(restRequest), getAckTimeout(restRequest));
         return channel -> client.execute(ILMActions.START, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestStopAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestStopAction.java
index 96954f4dbca42..d68bbe86fc7d9 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestStopAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestStopAction.java
@@ -34,9 +34,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        StopILMRequest request = new StopILMRequest();
-        request.ackTimeout(getAckTimeout(restRequest));
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        StopILMRequest request = new StopILMRequest(getMasterNodeTimeout(restRequest), getAckTimeout(restRequest));
         return channel -> client.execute(ILMActions.STOP, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java
index e222d8f6dd9d8..05f9fe7820baf 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -76,14 +75,6 @@ public DeleteLifecyclePolicyTask(Request request, ActionListener<AcknowledgedRes
             this.request = request;
         }
 
-        /**
-         * Used by the {@link ReservedClusterStateHandler} for ILM
-         * {@link ReservedLifecycleAction}
-         */
-        DeleteLifecyclePolicyTask(String policyName) {
-            this(new Request(policyName), null);
-        }
-
         @Override
         public ClusterState execute(ClusterState currentState) {
             String policyToDelete = request.getPolicyName();
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
index aa9da48eb8f8f..54b9cdca98393 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
@@ -80,6 +80,8 @@ public void testModeSnapshotRestore() throws Exception {
         client().execute(
             ILMActions.PUT,
             new PutLifecycleRequest(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
                 new LifecyclePolicy(
                     "ilm-policy",
                     Map.of("warm", new Phase("warm", TimeValue.timeValueHours(1), Map.of("readonly", new ReadOnlyAction())))
@@ -111,7 +113,7 @@ public void testModeSnapshotRestore() throws Exception {
             }
         });
 
-        assertAcked(client().execute(ILMActions.STOP, new StopILMRequest()).get());
+        assertAcked(client().execute(ILMActions.STOP, new StopILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)).get());
         assertAcked(client().execute(StopSLMAction.INSTANCE, new StopSLMAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)).get());
         assertBusy(() -> assertThat(ilmMode(), equalTo(OperationMode.STOPPED)));
         assertBusy(() -> assertThat(slmMode(), equalTo(OperationMode.STOPPED)));
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java
index 3e0f19246dfe3..e75a609d236ed 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java
@@ -32,7 +32,11 @@ public void testReservedStateHandler() {
         );
         assertEquals(ReservedLifecycleAction.NAME, putAction.reservedStateHandlerName().get());
 
-        DeleteLifecycleAction.Request request = new DeleteLifecycleAction.Request("my_timeseries_lifecycle2");
+        DeleteLifecycleAction.Request request = new DeleteLifecycleAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            "my_timeseries_lifecycle2"
+        );
         assertThat(putAction.modifiedKeys(request), containsInAnyOrder("my_timeseries_lifecycle2"));
     }
 }
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java
index e69e91192cf13..7f4e70bc1b306 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java
@@ -75,7 +75,17 @@ public void testReservedStateHandler() throws Exception {
             }""";
 
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)) {
-            PutLifecycleRequest request = PutLifecycleRequest.parseRequest("my_timeseries_lifecycle2", parser);
+            PutLifecycleRequest request = PutLifecycleRequest.parseRequest(new PutLifecycleRequest.Factory() {
+                @Override
+                public PutLifecycleRequest create(LifecyclePolicy lifecyclePolicy) {
+                    return new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, lifecyclePolicy);
+                }
+
+                @Override
+                public String getPolicyName() {
+                    return "my_timeseries_lifecycle2";
+                }
+            }, parser);
 
             assertThat(putAction.modifiedKeys(request), containsInAnyOrder("my_timeseries_lifecycle2"));
         }
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java
index 69ff6215aea01..8c0fede4c11dc 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportStopILMActionTests.java
@@ -52,7 +52,7 @@ public void testStopILMClusterStatePriorityIsImmediate() {
             new TaskId(randomLong() + ":" + randomLong()),
             emptyMap()
         );
-        StopILMRequest request = new StopILMRequest();
+        StopILMRequest request = new StopILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         transportStopILMAction.masterOperation(task, request, ClusterState.EMPTY_STATE, ActionListener.noop());
 
         verify(clusterService).submitUnbatchedStateUpdateTask(

From fc07f58be8bda3923c2fecb7fd3268b32be47032 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Wed, 29 May 2024 08:43:55 +0200
Subject: [PATCH 020/208] [Inference API] Use
 ModelConfigurations.SERVICE_SETTINGS as scope (#109072)

---
 .../elasticsearch/CustomElandInternalServiceSettings.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java
index ba98090c92522..c62855c09cff2 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/CustomElandInternalServiceSettings.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.inference.services.ServiceUtils;
 
@@ -45,8 +46,7 @@ public static Builder fromMap(Map<String, Object> map) {
 
         validateParameters(numAllocations, validationException, numThreads);
 
-        String modelId = ServiceUtils.extractRequiredString(map, MODEL_ID, "ServiceSettings", validationException); // TODO check if this is
-                                                                                                                    // the correct scope
+        String modelId = ServiceUtils.extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
 
         if (validationException.validationErrors().isEmpty() == false) {
             throw validationException;

From 0e03920560c2bbd486d205b2f0c06b44235e909f Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Wed, 29 May 2024 08:14:15 +0100
Subject: [PATCH 021/208] Clean up licensing APIs (#109121)

Fixes trappy timeouts related to #107984 and also cleans up all the
indirection via unnecessary request builders.
---
 .../license/ClusterStateLicenseService.java   |   4 +-
 .../license/GetBasicStatusRequest.java        |   5 +-
 .../license/GetBasicStatusRequestBuilder.java |  17 ---
 .../license/GetLicenseRequestBuilder.java     |  30 -----
 .../license/GetTrialStatusRequest.java        |   5 +-
 .../license/GetTrialStatusRequestBuilder.java |  17 ---
 .../license/LicensingClient.java              |  42 ------
 .../license/PostStartBasicRequest.java        |   5 +-
 .../license/PostStartBasicRequestBuilder.java |  22 ----
 .../license/PostStartTrialRequest.java        |   5 +-
 .../license/PostStartTrialRequestBuilder.java |  22 ----
 .../license/PutLicenseRequest.java            |   5 +-
 .../license/PutLicenseRequestBuilder.java     |  49 -------
 .../license/RestGetBasicStatus.java           |   6 +-
 .../license/RestGetLicenseAction.java         |   3 +-
 .../license/RestGetTrialStatus.java           |   6 +-
 .../license/RestPostStartBasicLicense.java    |   4 +-
 .../license/RestPostStartTrialLicense.java    |   3 +-
 .../license/RestPutLicenseAction.java         |   4 +-
 .../license/TransportDeleteLicenseAction.java |   6 +-
 .../internal/MutableLicenseService.java       |   3 +-
 .../xpack/license/GetLicenseRequest.java      |   5 +-
 .../AbstractLicensesIntegrationTestCase.java  |  17 +++
 ...lusterStateLicenseServiceClusterTests.java |  32 +++--
 .../ClusterStateLicenseServiceTests.java      |   4 +-
 .../license/LicenseFIPSTests.java             |   4 +-
 .../license/LicenseTLSTests.java              |   2 +-
 .../license/LicensesAcknowledgementTests.java |   4 +-
 .../license/LicensesManagerServiceTests.java  |   2 +-
 .../license/LicensesTransportTests.java       | 122 ++++++++----------
 .../license/StartBasicLicenseTests.java       |  65 ++++++----
 .../license/StartTrialLicenseTests.java       |  66 ++++------
 .../org/elasticsearch/license/TestUtils.java  |   5 +-
 .../xpack/core/XPackPluginTests.java          |   7 +-
 .../lucene/bwc/AbstractArchiveTestCase.java   |   3 +-
 .../lucene/bwc/ArchiveLicenseIntegTests.java  |  11 +-
 .../SearchableSnapshotsLicenseIntegTests.java |   7 +-
 37 files changed, 231 insertions(+), 388 deletions(-)
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequestBuilder.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequestBuilder.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequestBuilder.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequestBuilder.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequestBuilder.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequestBuilder.java

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java
index 7cab82559c7fc..b352a9abce886 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/ClusterStateLicenseService.java
@@ -303,8 +303,8 @@ public void triggered(SchedulerEngine.Event event) {
      * Remove license from the cluster state metadata
      */
     @Override
-    public void removeLicense(ActionListener<? extends AcknowledgedResponse> listener) {
-        final PostStartBasicRequest startBasicRequest = new PostStartBasicRequest().acknowledge(true);
+    public void removeLicense(TimeValue masterNodeTimeout, TimeValue ackTimeout, ActionListener<? extends AcknowledgedResponse> listener) {
+        final PostStartBasicRequest startBasicRequest = new PostStartBasicRequest(masterNodeTimeout, ackTimeout).acknowledge(true);
         @SuppressWarnings("unchecked")
         final StartBasicClusterTask task = new StartBasicClusterTask(
             logger,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequest.java
index 9e8e707db6b86..7ef61ca07f821 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequest.java
@@ -9,13 +9,14 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
 public class GetBasicStatusRequest extends MasterNodeReadRequest<GetBasicStatusRequest> {
 
-    public GetBasicStatusRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public GetBasicStatusRequest(TimeValue masterNodeTimeout) {
+        super(masterNodeTimeout);
     }
 
     public GetBasicStatusRequest(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequestBuilder.java
deleted file mode 100644
index 160b3648444aa..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequestBuilder.java
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-
-class GetBasicStatusRequestBuilder extends ActionRequestBuilder<GetBasicStatusRequest, GetBasicStatusResponse> {
-
-    GetBasicStatusRequestBuilder(ElasticsearchClient client) {
-        super(client, GetBasicStatusAction.INSTANCE, new GetBasicStatusRequest());
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequestBuilder.java
deleted file mode 100644
index 560ab2b861f8a..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseRequestBuilder.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
-
-public class GetLicenseRequestBuilder extends MasterNodeReadOperationRequestBuilder<
-    GetLicenseRequest,
-    GetLicenseResponse,
-    GetLicenseRequestBuilder> {
-
-    public GetLicenseRequestBuilder(ElasticsearchClient client) {
-        this(client, GetLicenseAction.INSTANCE);
-    }
-
-    /**
-     * Creates new get licenses request builder
-     *
-     * @param client elasticsearch client
-     */
-    public GetLicenseRequestBuilder(ElasticsearchClient client, GetLicenseAction action) {
-        super(client, action, new GetLicenseRequest());
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequest.java
index cae967058fb73..3643e4e7857ec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequest.java
@@ -9,13 +9,14 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
 public class GetTrialStatusRequest extends MasterNodeReadRequest<GetTrialStatusRequest> {
 
-    public GetTrialStatusRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public GetTrialStatusRequest(TimeValue masterNodeTimeout) {
+        super(masterNodeTimeout);
     }
 
     public GetTrialStatusRequest(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequestBuilder.java
deleted file mode 100644
index 6c51def015082..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequestBuilder.java
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-
-class GetTrialStatusRequestBuilder extends ActionRequestBuilder<GetTrialStatusRequest, GetTrialStatusResponse> {
-
-    GetTrialStatusRequestBuilder(ElasticsearchClient client) {
-        super(client, GetTrialStatusAction.INSTANCE, new GetTrialStatusRequest());
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
deleted file mode 100644
index 5396d45126bb5..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.client.internal.ElasticsearchClient;
-
-public class LicensingClient {
-
-    private final ElasticsearchClient client;
-
-    public LicensingClient(ElasticsearchClient client) {
-        this.client = client;
-    }
-
-    public PutLicenseRequestBuilder preparePutLicense(License license) {
-        return new PutLicenseRequestBuilder(client).setLicense(license);
-    }
-
-    public GetLicenseRequestBuilder prepareGetLicense() {
-        return new GetLicenseRequestBuilder(client);
-    }
-
-    public PostStartTrialRequestBuilder preparePostStartTrial() {
-        return new PostStartTrialRequestBuilder(client);
-    }
-
-    public GetTrialStatusRequestBuilder prepareGetStartTrial() {
-        return new GetTrialStatusRequestBuilder(client);
-    }
-
-    public PostStartBasicRequestBuilder preparePostStartBasic() {
-        return new PostStartBasicRequestBuilder(client);
-    }
-
-    public GetBasicStatusRequestBuilder prepareGetStartBasic() {
-        return new GetBasicStatusRequestBuilder(client);
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequest.java
index 7e9b0ebf44bee..e3f77ca97cd1c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequest.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -16,8 +17,8 @@ public class PostStartBasicRequest extends AcknowledgedRequest<PostStartBasicReq
 
     private boolean acknowledge = false;
 
-    public PostStartBasicRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+    public PostStartBasicRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+        super(masterNodeTimeout, ackTimeout);
     }
 
     public PostStartBasicRequest(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequestBuilder.java
deleted file mode 100644
index 7e926c0560c05..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequestBuilder.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-
-class PostStartBasicRequestBuilder extends ActionRequestBuilder<PostStartBasicRequest, PostStartBasicResponse> {
-
-    PostStartBasicRequestBuilder(ElasticsearchClient client) {
-        super(client, PostStartBasicAction.INSTANCE, new PostStartBasicRequest());
-    }
-
-    public PostStartBasicRequestBuilder setAcknowledge(boolean acknowledge) {
-        request.acknowledge(acknowledge);
-        return this;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequest.java
index 5d7ee5648bd10..835d75180a821 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequest.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -18,8 +19,8 @@ public class PostStartTrialRequest extends MasterNodeRequest<PostStartTrialReque
     private boolean acknowledge = false;
     private String type;
 
-    public PostStartTrialRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public PostStartTrialRequest(TimeValue masterNodeTimeout) {
+        super(masterNodeTimeout);
     }
 
     public PostStartTrialRequest(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequestBuilder.java
deleted file mode 100644
index 826f557fa9bfb..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequestBuilder.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-
-class PostStartTrialRequestBuilder extends ActionRequestBuilder<PostStartTrialRequest, PostStartTrialResponse> {
-
-    PostStartTrialRequestBuilder(ElasticsearchClient client) {
-        super(client, PostStartTrialAction.INSTANCE, new PostStartTrialRequest());
-    }
-
-    public PostStartTrialRequestBuilder setAcknowledge(boolean acknowledge) {
-        request.acknowledge(acknowledge);
-        return this;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequest.java
index d97b0aa7f4ff6..ed74ccc69731c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequest.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
@@ -27,8 +28,8 @@ public PutLicenseRequest(StreamInput in) throws IOException {
         acknowledge = in.readBoolean();
     }
 
-    public PutLicenseRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+    public PutLicenseRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+        super(masterNodeTimeout, ackTimeout);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequestBuilder.java
deleted file mode 100644
index 532e63e24d09e..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseRequestBuilder.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.license;
-
-import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
-import org.elasticsearch.xcontent.XContentType;
-
-/**
- * Register license request builder
- */
-public class PutLicenseRequestBuilder extends AcknowledgedRequestBuilder<PutLicenseRequest, PutLicenseResponse, PutLicenseRequestBuilder> {
-
-    /**
-     * Constructs register license request
-     *
-     * @param client elasticsearch client
-     */
-    public PutLicenseRequestBuilder(ElasticsearchClient client) {
-        super(client, PutLicenseAction.INSTANCE, new PutLicenseRequest());
-    }
-
-    /**
-     * Sets the license
-     *
-     * @param license license
-     * @return this builder
-     */
-    public PutLicenseRequestBuilder setLicense(License license) {
-        request.license(license);
-        return this;
-    }
-
-    public PutLicenseRequestBuilder setLicense(BytesReference licenseSource, XContentType xContentType) {
-        request.license(licenseSource, xContentType);
-        return this;
-    }
-
-    public PutLicenseRequestBuilder setAcknowledge(boolean acknowledge) {
-        request.acknowledge(acknowledge);
-        return this;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java
index c1efbc7513e4a..e0428c0ff2039 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.util.List;
@@ -29,8 +30,9 @@ public List<Route> routes() {
     }
 
     @Override
-    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        return channel -> new GetBasicStatusRequestBuilder(client).execute(new RestToXContentListener<>(channel));
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+        final var request = new GetBasicStatusRequest(RestUtils.getMasterNodeTimeout(restRequest));
+        return channel -> client.execute(GetBasicStatusAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
index 75826bc0b1828..4240119df457d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestBuilderListener;
@@ -84,7 +85,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         }
 
         final ToXContent.Params params = new ToXContent.DelegatingMapParams(overrideParams, request);
-        GetLicenseRequest getLicenseRequest = new GetLicenseRequest();
+        GetLicenseRequest getLicenseRequest = new GetLicenseRequest(RestUtils.getMasterNodeTimeout(request));
         getLicenseRequest.local(request.paramAsBoolean("local", getLicenseRequest.local()));
         return channel -> client.admin()
             .cluster()
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java
index b453551749d8b..e42db16ded401 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.util.List;
@@ -29,8 +30,9 @@ public List<Route> routes() {
     }
 
     @Override
-    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
-        return channel -> new GetTrialStatusRequestBuilder(client).execute(new RestToXContentListener<>(channel));
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+        final var request = new GetTrialStatusRequest(RestUtils.getMasterNodeTimeout(restRequest));
+        return channel -> client.execute(GetTrialStatusAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java
index 1a1a568393789..64556bcf69ecf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java
@@ -33,10 +33,8 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        PostStartBasicRequest startBasicRequest = new PostStartBasicRequest();
+        PostStartBasicRequest startBasicRequest = new PostStartBasicRequest(getMasterNodeTimeout(request), getAckTimeout(request));
         startBasicRequest.acknowledge(request.paramAsBoolean("acknowledge", false));
-        startBasicRequest.ackTimeout(getAckTimeout(request));
-        startBasicRequest.masterNodeTimeout(getMasterNodeTimeout(request));
         return channel -> client.execute(
             PostStartBasicAction.INSTANCE,
             startBasicRequest,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java
index 6ae1719383afb..56fae76c2e2b8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -34,7 +35,7 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        PostStartTrialRequest startTrialRequest = new PostStartTrialRequest();
+        PostStartTrialRequest startTrialRequest = new PostStartTrialRequest(RestUtils.getMasterNodeTimeout(request));
         startTrialRequest.setType(request.param("type", License.LicenseType.TRIAL.getTypeName()));
         startTrialRequest.acknowledge(request.paramAsBoolean("acknowledge", false));
         return channel -> client.execute(PostStartTrialAction.INSTANCE, startTrialRequest, new RestBuilderListener<>(channel) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java
index 2d86a75947ba0..0798be6e53a14 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java
@@ -44,11 +44,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         if (request.hasContent() == false) {
             throw new IllegalArgumentException("The license must be provided in the request body");
         }
-        PutLicenseRequest putLicenseRequest = new PutLicenseRequest();
+        PutLicenseRequest putLicenseRequest = new PutLicenseRequest(getMasterNodeTimeout(request), getAckTimeout(request));
         putLicenseRequest.license(request.content(), request.getXContentType());
         putLicenseRequest.acknowledge(request.paramAsBoolean("acknowledge", false));
-        putLicenseRequest.ackTimeout(getAckTimeout(request));
-        putLicenseRequest.masterNodeTimeout(getMasterNodeTimeout(request));
 
         if (License.LicenseType.isBasic(putLicenseRequest.license().type())) {
             throw new IllegalArgumentException(
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
index 7ac59e1dc327a..5739655c88eea 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
@@ -63,6 +63,10 @@ protected void masterOperation(
         ClusterState state,
         final ActionListener<AcknowledgedResponse> listener
     ) throws ElasticsearchException {
-        licenseService.removeLicense(listener.map(r -> AcknowledgedResponse.of(r.isAcknowledged())));
+        licenseService.removeLicense(
+            request.masterNodeTimeout(),
+            request.ackTimeout(),
+            listener.map(r -> AcknowledgedResponse.of(r.isAcknowledged()))
+        );
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/internal/MutableLicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/internal/MutableLicenseService.java
index 72650daa1215e..8632e60c2ee4f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/internal/MutableLicenseService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/internal/MutableLicenseService.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.component.LifecycleComponent;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.LicensesMetadata;
 import org.elasticsearch.license.PostStartBasicRequest;
@@ -35,7 +36,7 @@ public interface MutableLicenseService extends LicenseService, LifecycleComponen
      * Removes the current license. Implementations should remove the current license and ensure that attempts to read returns
      * {@link LicensesMetadata#LICENSE_TOMBSTONE} if a license was removed. Additionally the {@link XPackLicenseState} must be updated.
      */
-    void removeLicense(ActionListener<? extends AcknowledgedResponse> listener);
+    void removeLicense(TimeValue masterNodeTimeout, TimeValue ackTimeout, ActionListener<? extends AcknowledgedResponse> listener);
 
     /**
      * Installs a basic license.
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/license/GetLicenseRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/license/GetLicenseRequest.java
index ea4e53aced5fe..b05b87b699a72 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/license/GetLicenseRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/license/GetLicenseRequest.java
@@ -9,13 +9,14 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
 public class GetLicenseRequest extends MasterNodeReadRequest<GetLicenseRequest> {
 
-    public GetLicenseRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public GetLicenseRequest(TimeValue masterNodeTimeout) {
+        super(masterNodeTimeout);
     }
 
     public GetLicenseRequest(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java
index aa3a4f44e2f12..a09f7d5ca3f52 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
@@ -100,4 +101,20 @@ protected void assertLicenseActive(boolean active) throws Exception {
         });
     }
 
+    protected static GetLicenseResponse getLicense() {
+        return safeGet(clusterAdmin().execute(GetLicenseAction.INSTANCE, new GetLicenseRequest(TEST_REQUEST_TIMEOUT)));
+    }
+
+    protected static GetTrialStatusResponse getTrialStatus() {
+        return safeGet(clusterAdmin().execute(GetTrialStatusAction.INSTANCE, new GetTrialStatusRequest(TEST_REQUEST_TIMEOUT)));
+    }
+
+    protected static PostStartBasicResponse startBasic() {
+        return safeGet(
+            clusterAdmin().execute(
+                PostStartBasicAction.INSTANCE,
+                new PostStartBasicRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).acknowledge(true)
+            )
+        );
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceClusterTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceClusterTests.java
index 338c1ebfb4e21..00bf1c7fe174f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceClusterTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceClusterTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 
 import java.util.Set;
@@ -40,18 +41,22 @@ public void testClusterRestartWithLicense() throws Exception {
         ensureGreen();
 
         logger.info("--> put signed license");
-        LicensingClient licensingClient = new LicensingClient(client());
         License license = TestUtils.generateSignedLicense(TimeValue.timeValueMinutes(1));
         putLicense(license);
-        assertThat(licensingClient.prepareGetLicense().get().license(), equalTo(license));
+        assertThat(
+            client().execute(GetLicenseAction.INSTANCE, new GetLicenseRequest(TEST_REQUEST_TIMEOUT)).get().license(),
+            equalTo(license)
+        );
         assertOperationMode(license.operationMode());
 
         logger.info("--> restart all nodes");
         internalCluster().fullRestart();
         ensureYellow();
-        licensingClient = new LicensingClient(client());
         logger.info("--> get and check signed license");
-        assertThat(licensingClient.prepareGetLicense().get().license(), equalTo(license));
+        assertThat(
+            client().execute(GetLicenseAction.INSTANCE, new GetLicenseRequest(TEST_REQUEST_TIMEOUT)).get().license(),
+            equalTo(license)
+        );
         logger.info("--> remove licenses");
 
         assertAcked(
@@ -62,9 +67,12 @@ public void testClusterRestartWithLicense() throws Exception {
 
         logger.info("--> restart all nodes");
         internalCluster().fullRestart();
-        licensingClient = new LicensingClient(client());
         ensureYellow();
-        assertTrue(License.LicenseType.isBasic(licensingClient.prepareGetLicense().get().license().type()));
+        assertTrue(
+            License.LicenseType.isBasic(
+                client().execute(GetLicenseAction.INSTANCE, new GetLicenseRequest(TEST_REQUEST_TIMEOUT)).get().license().type()
+            )
+        );
         assertOperationMode(License.OperationMode.BASIC);
 
         wipeAllLicenses();
@@ -103,15 +111,19 @@ public void testClusterRestartWithOldSignature() throws Exception {
         ensureGreen();
         assertLicenseActive(true);
         putLicense(TestUtils.generateSignedLicenseOldSignature());
-        LicensingClient licensingClient = new LicensingClient(client());
-        assertThat(licensingClient.prepareGetLicense().get().license().version(), equalTo(License.VERSION_START_DATE));
+        assertThat(
+            client().execute(GetLicenseAction.INSTANCE, new GetLicenseRequest(TEST_REQUEST_TIMEOUT)).get().license().version(),
+            equalTo(License.VERSION_START_DATE)
+        );
         logger.info("--> restart node");
         internalCluster().fullRestart(); // restart so that license is updated
         ensureYellow();
         logger.info("--> await node for enabled");
         assertLicenseActive(true);
-        licensingClient = new LicensingClient(client());
-        assertThat(licensingClient.prepareGetLicense().get().license().version(), equalTo(License.VERSION_CURRENT)); // license updated
+        assertThat(
+            client().execute(GetLicenseAction.INSTANCE, new GetLicenseRequest(TEST_REQUEST_TIMEOUT)).get().license().version(),
+            equalTo(License.VERSION_CURRENT)
+        ); // license updated
         internalCluster().fullRestart(); // restart once more and verify updated license is active
         ensureYellow();
         logger.info("--> await node for enabled");
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceTests.java
index 33f162a06f350..c0c7c5c59d24b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/ClusterStateLicenseServiceTests.java
@@ -202,7 +202,7 @@ public void testStartBasicStartsNewLicenseIfFieldsDifferent() throws Exception {
             assertThat(response.getStatus(), equalTo(PostStartBasicResponse.Status.GENERATED_BASIC));
         };
         final PlainActionFuture<PostStartBasicResponse> future = new PlainActionFuture<>();
-        service.startBasicLicense(new PostStartBasicRequest(), future);
+        service.startBasicLicense(new PostStartBasicRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT), future);
 
         if (future.isDone()) {
             // If validation failed, the future might be done without calling the updater task.
@@ -289,7 +289,7 @@ private void tryRegisterLicense(Settings baseSettings, License license, Consumer
             new FeatureService(List.of())
         );
 
-        final PutLicenseRequest request = new PutLicenseRequest();
+        final PutLicenseRequest request = new PutLicenseRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.license(toSpec(license), XContentType.JSON);
         final PlainActionFuture<PutLicenseResponse> future = new PlainActionFuture<>();
         service.registerLicense(request, future);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java
index 212ca265ac477..39bbaa0e53872 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseFIPSTests.java
@@ -21,7 +21,7 @@ public class LicenseFIPSTests extends AbstractClusterStateLicenseServiceTestCase
 
     public void testFIPSCheckWithAllowedLicense() throws Exception {
         License newLicense = TestUtils.generateSignedLicense(randomFrom("trial", "platinum"), TimeValue.timeValueHours(24L));
-        PutLicenseRequest request = new PutLicenseRequest();
+        PutLicenseRequest request = new PutLicenseRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.acknowledge(true);
         request.license(newLicense);
         Settings settings = Settings.builder()
@@ -45,7 +45,7 @@ public void testFIPSCheckWithAllowedLicense() throws Exception {
 
     public void testFIPSCheckWithoutAllowedLicense() throws Exception {
         License newLicense = TestUtils.generateSignedLicense(randomFrom("gold", "standard"), TimeValue.timeValueHours(24L));
-        PutLicenseRequest request = new PutLicenseRequest();
+        PutLicenseRequest request = new PutLicenseRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.acknowledge(true);
         request.license(newLicense);
         Settings settings = Settings.builder()
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTLSTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTLSTests.java
index e72c0261e93ca..736f355c542e5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTLSTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTLSTests.java
@@ -31,7 +31,7 @@ public class LicenseTLSTests extends AbstractClusterStateLicenseServiceTestCase
 
     public void testApplyLicenseInDevMode() throws Exception {
         License newLicense = TestUtils.generateSignedLicense(randomFrom("gold", "platinum"), TimeValue.timeValueHours(24L));
-        PutLicenseRequest request = new PutLicenseRequest();
+        PutLicenseRequest request = new PutLicenseRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.acknowledge(true);
         request.license(newLicense);
         Settings settings = Settings.builder().put("xpack.security.enabled", true).build();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesAcknowledgementTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesAcknowledgementTests.java
index f23262e8b4d42..5a0b175db24c8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesAcknowledgementTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesAcknowledgementTests.java
@@ -29,14 +29,14 @@ public void testAcknowledgment() throws Exception {
         // try installing a signed license
         long issueDate = System.currentTimeMillis() - TimeValue.timeValueHours(24 * 2).getMillis();
         License signedLicense = TestUtils.generateSignedLicense("trial", License.VERSION_CURRENT, issueDate, timeValueHours(10));
-        PutLicenseRequest putLicenseRequest = new PutLicenseRequest().license(signedLicense);
+        PutLicenseRequest putLicenseRequest = new PutLicenseRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).license(signedLicense);
         // ensure acknowledgement message was part of the response
         licenseService.registerLicense(putLicenseRequest, new AssertingLicensesUpdateResponse(false, LicensesStatus.VALID, true));
         assertThat(licenseService.getLicense(), not(signedLicense));
         verify(clusterService, times(0)).submitUnbatchedStateUpdateTask(any(String.class), any(ClusterStateUpdateTask.class));
 
         // try installing a signed license with acknowledgement
-        putLicenseRequest = new PutLicenseRequest().license(signedLicense).acknowledge(true);
+        putLicenseRequest = new PutLicenseRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).license(signedLicense).acknowledge(true);
         // ensure license was installed and no acknowledgment message was returned
         licenseService.registerLicense(putLicenseRequest, new AssertingLicensesUpdateResponse(true, LicensesStatus.VALID, false));
         verify(clusterService, times(1)).submitUnbatchedStateUpdateTask(any(String.class), any(ClusterStateUpdateTask.class));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java
index 7207a823c543b..0ea55617d86b2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java
@@ -127,7 +127,7 @@ public void testRemoveLicenses() throws Exception {
     private void removeAndAckSignedLicenses(final MutableLicenseService licenseService) {
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicBoolean success = new AtomicBoolean(false);
-        licenseService.removeLicense(new ActionListener<PostStartBasicResponse>() {
+        licenseService.removeLicense(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, new ActionListener<PostStartBasicResponse>() {
             @Override
             public void onResponse(PostStartBasicResponse postStartBasicResponse) {
                 if (postStartBasicResponse.isAcknowledged()) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
index 7e9152c0cf18e..afecf0da5d55d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesTransportTests.java
@@ -6,13 +6,13 @@
  */
 package org.elasticsearch.license;
 
-import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.LicensesStatus;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -24,6 +24,9 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.function.UnaryOperator;
 
 import static org.elasticsearch.license.TestUtils.dateMath;
 import static org.elasticsearch.license.TestUtils.generateExpiredNonBasicLicense;
@@ -59,12 +62,7 @@ public void testEmptyGetLicense() throws Exception {
         // basic license is added async, we should wait for it
         assertBusy(() -> {
             try {
-                final ActionFuture<GetLicenseResponse> getLicenseFuture = new GetLicenseRequestBuilder(
-                    clusterAdmin(),
-                    GetLicenseAction.INSTANCE
-                ).execute();
-                final GetLicenseResponse getLicenseResponse;
-                getLicenseResponse = getLicenseFuture.get();
+                final GetLicenseResponse getLicenseResponse = getLicense();
                 assertNotNull(getLicenseResponse.license());
                 assertThat(getLicenseResponse.license().operationMode(), equalTo(License.OperationMode.BASIC));
             } catch (Exception e) {
@@ -77,15 +75,12 @@ public void testPutLicense() throws Exception {
         License signedLicense = generateSignedLicense(TimeValue.timeValueMinutes(2));
 
         // put license
-        PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(clusterAdmin()).setLicense(signedLicense)
-            .setAcknowledge(true);
-        PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get();
+        final var putLicenseResponse = putLicense(plr -> plr.license(signedLicense).acknowledge(true));
         assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
 
         // get and check license
-        GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
-        assertThat(getLicenseResponse.license(), equalTo(signedLicense));
+        assertThat(getLicense().license(), equalTo(signedLicense));
     }
 
     public void testPutLicenseFromString() throws Exception {
@@ -93,17 +88,14 @@ public void testPutLicenseFromString() throws Exception {
         String licenseString = TestUtils.dumpLicense(signedLicense);
 
         // put license source
-        PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(clusterAdmin()).setLicense(
-            new BytesArray(licenseString.getBytes(StandardCharsets.UTF_8)),
-            XContentType.JSON
-        ).setAcknowledge(true);
-        PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get();
+        final var putLicenseResponse = putLicense(
+            plr -> plr.license(new BytesArray(licenseString.getBytes(StandardCharsets.UTF_8)), XContentType.JSON).acknowledge(true)
+        );
         assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
 
         // get and check license
-        GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
-        assertThat(getLicenseResponse.license(), equalTo(signedLicense));
+        assertThat(getLicense().license(), equalTo(signedLicense));
     }
 
     public void testPutInvalidLicense() throws Exception {
@@ -115,81 +107,57 @@ public void testPutInvalidLicense() throws Exception {
             .expiryDate(signedLicense.expiryDate() + 10 * 24 * 60 * 60 * 1000L)
             .validate()
             .build();
-
-        PutLicenseRequestBuilder builder = new PutLicenseRequestBuilder(clusterAdmin());
-        builder.setLicense(tamperedLicense);
-
-        // try to put license (should be invalid)
-        final PutLicenseResponse putLicenseResponse = builder.get();
+        final var putLicenseResponse = putLicense(plr -> plr.license(tamperedLicense).acknowledge(randomBoolean()));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.INVALID));
 
         // try to get invalid license
-        GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
-        assertThat(getLicenseResponse.license(), not(tamperedLicense));
+        assertThat(getLicense().license(), not(tamperedLicense));
     }
 
     public void testPutBasicLicenseIsInvalid() throws Exception {
         License signedLicense = generateSignedLicense("basic", License.VERSION_CURRENT, -1, TimeValue.timeValueMinutes(2));
 
-        PutLicenseRequestBuilder builder = new PutLicenseRequestBuilder(clusterAdmin());
-        builder.setLicense(signedLicense);
-
         // try to put license (should be invalid)
-        IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, builder::get);
+        final var putLicenseFuture = putLicenseFuture(plr -> plr.license(signedLicense).acknowledge(randomBoolean()));
+        IllegalArgumentException iae = expectThrows(ExecutionException.class, IllegalArgumentException.class, putLicenseFuture::get);
         assertEquals(iae.getMessage(), "Registering basic licenses is not allowed.");
 
         // try to get invalid license
-        GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
-        assertThat(getLicenseResponse.license(), not(signedLicense));
+        assertThat(getLicense().license(), not(signedLicense));
     }
 
     public void testPutExpiredLicense() throws Exception {
         License expiredLicense = generateExpiredNonBasicLicense();
-        PutLicenseRequestBuilder builder = new PutLicenseRequestBuilder(clusterAdmin());
-        builder.setLicense(expiredLicense);
-        PutLicenseResponse putLicenseResponse = builder.get();
+        final var putLicenseResponse = putLicense(plr -> plr.license(expiredLicense).acknowledge(randomBoolean()));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.EXPIRED));
         // get license should not return the expired license
-        GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
-        assertThat(getLicenseResponse.license(), not(expiredLicense));
+        assertThat(getLicense().license(), not(expiredLicense));
     }
 
     public void testPutLicensesSimple() throws Exception {
         License goldSignedLicense = generateSignedLicense("gold", TimeValue.timeValueMinutes(5));
-        PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(clusterAdmin()).setLicense(goldSignedLicense)
-            .setAcknowledge(true);
-        PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get();
-        assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
-        GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
-        assertThat(getLicenseResponse.license(), equalTo(goldSignedLicense));
+        final var putGoldLicenseResponse = putLicense(plr -> plr.license(goldSignedLicense).acknowledge(true));
+        assertThat(putGoldLicenseResponse.status(), equalTo(LicensesStatus.VALID));
+        assertThat(getLicense().license(), equalTo(goldSignedLicense));
 
         License platinumSignedLicense = generateSignedLicense("platinum", TimeValue.timeValueMinutes(2));
-        putLicenseRequestBuilder.setLicense(platinumSignedLicense);
-        putLicenseResponse = putLicenseRequestBuilder.get();
-        assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
-        assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
-        getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
-        assertThat(getLicenseResponse.license(), equalTo(platinumSignedLicense));
+        final var putPlatinumLicenseResponse = putLicense(plr -> plr.license(platinumSignedLicense).acknowledge(true));
+        assertThat(putPlatinumLicenseResponse.isAcknowledged(), equalTo(true));
+        assertThat(putPlatinumLicenseResponse.status(), equalTo(LicensesStatus.VALID));
+        assertThat(getLicense().license(), equalTo(platinumSignedLicense));
     }
 
     public void testRemoveLicensesSimple() throws Exception {
         License goldLicense = generateSignedLicense("gold", TimeValue.timeValueMinutes(5));
-        PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(clusterAdmin()).setLicense(goldLicense)
-            .setAcknowledge(true);
-        PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get();
+        PutLicenseResponse putLicenseResponse = putLicense(plr -> plr.license(goldLicense).acknowledge(true));
         assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
-        GetLicenseResponse getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
-        assertThat(getLicenseResponse.license(), equalTo(goldLicense));
+        assertThat(getLicense().license(), equalTo(goldLicense));
         // delete all licenses
-        AcknowledgedResponse deleteLicenseResponse = clusterAdmin().execute(
-            TransportDeleteLicenseAction.TYPE,
-            new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
-        ).get();
+        AcknowledgedResponse deleteLicenseResponse = deleteLicense();
         assertThat(deleteLicenseResponse.isAcknowledged(), equalTo(true));
         // get licenses (expected no licenses)
-        getLicenseResponse = new GetLicenseRequestBuilder(clusterAdmin(), GetLicenseAction.INSTANCE).get();
-        assertTrue(License.LicenseType.isBasic(getLicenseResponse.license().type()));
+        assertTrue(License.LicenseType.isBasic(getLicense().license().type()));
     }
 
     public void testLicenseIsRejectWhenStartDateLaterThanNow() throws Exception {
@@ -206,9 +174,7 @@ public void testLicenseIsRejectWhenStartDateLaterThanNow() throws Exception {
             .maxNodes(5);
         License license = TestUtils.generateSignedLicense(builder);
 
-        PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(clusterAdmin()).setLicense(license)
-            .setAcknowledge(true);
-        PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get();
+        PutLicenseResponse putLicenseResponse = putLicense(plr -> plr.license(license).acknowledge(true));
         assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.INVALID));
     }
@@ -227,10 +193,32 @@ public void testLicenseIsAcceptedWhenStartDateBeforeThanNow() throws Exception {
             .maxNodes(5);
         License license = TestUtils.generateSignedLicense(builder);
 
-        PutLicenseRequestBuilder putLicenseRequestBuilder = new PutLicenseRequestBuilder(clusterAdmin()).setLicense(license)
-            .setAcknowledge(true);
-        PutLicenseResponse putLicenseResponse = putLicenseRequestBuilder.get();
+        PutLicenseResponse putLicenseResponse = putLicense(plr -> plr.license(license).acknowledge(true));
         assertThat(putLicenseResponse.isAcknowledged(), equalTo(true));
         assertThat(putLicenseResponse.status(), equalTo(LicensesStatus.VALID));
     }
+
+    private GetLicenseResponse getLicense() {
+        return safeGet(clusterAdmin().execute(GetLicenseAction.INSTANCE, new GetLicenseRequest(TEST_REQUEST_TIMEOUT)));
+    }
+
+    private Future<PutLicenseResponse> putLicenseFuture(UnaryOperator<PutLicenseRequest> onRequest) {
+        return clusterAdmin().execute(
+            PutLicenseAction.INSTANCE,
+            onRequest.apply(new PutLicenseRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
+        );
+    }
+
+    private PutLicenseResponse putLicense(UnaryOperator<PutLicenseRequest> onRequest) {
+        return safeGet(putLicenseFuture(onRequest));
+    }
+
+    private AcknowledgedResponse deleteLicense() {
+        return safeGet(
+            clusterAdmin().execute(
+                TransportDeleteLicenseAction.TYPE,
+                new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+            )
+        );
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java
index cdaa5a2f86be7..be46ccf5e6ce6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java
@@ -20,6 +20,7 @@
 
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.elasticsearch.test.NodeRoles.addRoles;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 
 @ESIntegTestCase.ClusterScope(scope = SUITE)
 public class StartBasicLicenseTests extends AbstractLicensesIntegrationTestCase {
@@ -38,38 +39,28 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testStartBasicLicense() throws Exception {
-        LicensingClient licensingClient = new LicensingClient(client());
-        License license = TestUtils.generateSignedLicense("trial", License.VERSION_CURRENT, -1, TimeValue.timeValueHours(24));
-        licensingClient.preparePutLicense(license).get();
+        generateAndPutTestLicense();
 
-        assertBusy(() -> {
-            GetLicenseResponse getLicenseResponse = licensingClient.prepareGetLicense().get();
-            assertEquals("trial", getLicenseResponse.license().type());
-        });
+        assertBusy(() -> assertEquals("trial", getLicense().license().type()));
 
-        GetBasicStatusResponse response = licensingClient.prepareGetStartBasic().get();
-        assertTrue(response.isEligibleToStartBasic());
+        assertTrue(getBasicStatus().isEligibleToStartBasic());
 
-        PostStartBasicResponse startResponse = licensingClient.preparePostStartBasic().setAcknowledge(true).get();
+        PostStartBasicResponse startResponse = startBasic(true);
         assertTrue(startResponse.isAcknowledged());
         assertTrue(startResponse.getStatus().isBasicStarted());
 
-        assertBusy(() -> {
-            GetLicenseResponse currentLicense = licensingClient.prepareGetLicense().get();
-            assertEquals("basic", currentLicense.license().type());
-        });
+        assertBusy(() -> assertEquals("basic", getLicense().license().type()));
 
-        long expirationMillis = licensingClient.prepareGetLicense().get().license().expiryDate();
+        long expirationMillis = getLicense().license().expiryDate();
         assertEquals(LicenseSettings.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS, expirationMillis);
 
-        GetLicenseResponse licenseResponse = licensingClient.prepareGetLicense().get();
+        GetLicenseResponse licenseResponse = getLicense();
         assertEquals("basic", licenseResponse.license().type());
         assertEquals(XPackInfoResponse.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS, licenseResponse.license().expiryDate());
 
-        GetBasicStatusResponse response4 = licensingClient.prepareGetStartBasic().get();
-        assertFalse(response4.isEligibleToStartBasic());
+        assertFalse(getBasicStatus().isEligibleToStartBasic());
 
-        PostStartBasicResponse response5 = licensingClient.preparePostStartBasic().setAcknowledge(true).get();
+        PostStartBasicResponse response5 = startBasic(true);
         assertEquals(403, response5.status().getStatus());
         assertFalse(response5.getStatus().isBasicStarted());
         assertTrue(response5.isAcknowledged());
@@ -77,16 +68,11 @@ public void testStartBasicLicense() throws Exception {
     }
 
     public void testUnacknowledgedStartBasicLicense() throws Exception {
-        LicensingClient licensingClient = new LicensingClient(client());
-        License license = TestUtils.generateSignedLicense("trial", License.VERSION_CURRENT, -1, TimeValue.timeValueHours(24));
-        licensingClient.preparePutLicense(license).get();
+        generateAndPutTestLicense();
 
-        assertBusy(() -> {
-            GetLicenseResponse getLicenseResponse = licensingClient.prepareGetLicense().get();
-            assertEquals("trial", getLicenseResponse.license().type());
-        });
+        assertBusy(() -> assertEquals("trial", getLicense().license().type()));
 
-        PostStartBasicResponse response = licensingClient.preparePostStartBasic().get();
+        PostStartBasicResponse response = startBasic(false);
         assertEquals(200, response.status().getStatus());
         assertFalse(response.isAcknowledged());
         assertFalse(response.getStatus().isBasicStarted());
@@ -97,4 +83,29 @@ public void testUnacknowledgedStartBasicLicense() throws Exception {
             response.getAcknowledgeMessage()
         );
     }
+
+    private static GetBasicStatusResponse getBasicStatus() {
+        return safeGet(clusterAdmin().execute(GetBasicStatusAction.INSTANCE, new GetBasicStatusRequest(TEST_REQUEST_TIMEOUT)));
+    }
+
+    private static PostStartBasicResponse startBasic(boolean acknowledged) {
+        return safeGet(
+            clusterAdmin().execute(
+                PostStartBasicAction.INSTANCE,
+                new PostStartBasicRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).acknowledge(acknowledged)
+            )
+        );
+    }
+
+    private static void generateAndPutTestLicense() throws Exception {
+        final var license = TestUtils.generateSignedLicense("trial", License.VERSION_CURRENT, -1, TimeValue.timeValueHours(24));
+        assertAcked(
+            safeGet(
+                client().execute(
+                    PutLicenseAction.INSTANCE,
+                    new PutLicenseRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).license(license).acknowledge(randomBoolean())
+                )
+            )
+        );
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java
index bd9f058f88100..cd69ea7d1dc62 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartTrialLicenseTests.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.license;
 
+import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
@@ -15,9 +16,11 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Set;
+import java.util.function.UnaryOperator;
 
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.elasticsearch.test.NodeRoles.addRoles;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 
 @ESIntegTestCase.ClusterScope(scope = SUITE)
@@ -37,74 +40,55 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testStartTrial() throws Exception {
-        LicensingClient licensingClient = new LicensingClient(client());
         ensureStartingWithBasic();
 
-        GetTrialStatusResponse response = licensingClient.prepareGetStartTrial().get();
-        assertTrue(response.isEligibleToStartTrial());
+        assertTrue(getTrialStatus().isEligibleToStartTrial());
+
+        License.LicenseType type = randomFrom(LicenseSettings.VALID_TRIAL_TYPES);
 
         // Test that starting will fail without acknowledgement
-        PostStartTrialRequestBuilder builder = licensingClient.preparePostStartTrial();
-        builder.request().setType(randomFrom(LicenseSettings.VALID_TRIAL_TYPES).getTypeName());
-        PostStartTrialResponse response2 = builder.get();
+        final PostStartTrialResponse response2 = startTrial(pstr -> pstr.setType(type.getTypeName()));
         assertEquals(200, response2.getStatus().getRestStatus().getStatus());
         assertFalse(response2.getStatus().isTrialStarted());
         assertEquals("Operation failed: Needs acknowledgement.", response2.getStatus().getErrorMessage());
 
-        assertBusy(() -> {
-            GetLicenseResponse getLicenseResponse = licensingClient.prepareGetLicense().get();
-            assertEquals("basic", getLicenseResponse.license().type());
-        });
-
-        License.LicenseType type = randomFrom(LicenseSettings.VALID_TRIAL_TYPES);
+        assertBusy(() -> assertEquals("basic", getLicense().license().type()));
 
-        PostStartTrialRequestBuilder builder2 = licensingClient.preparePostStartTrial();
-        builder2.setAcknowledge(true);
-        builder2.request().setType(type.getTypeName());
-        PostStartTrialResponse response3 = builder2.get();
+        final PostStartTrialResponse response3 = startTrial(pstr -> pstr.setType(type.getTypeName()).acknowledge(true));
         assertEquals(200, response3.getStatus().getRestStatus().getStatus());
         assertTrue(response3.getStatus().isTrialStarted());
 
-        assertBusy(() -> {
-            GetLicenseResponse postTrialLicenseResponse = licensingClient.prepareGetLicense().get();
-            assertEquals(type.getTypeName(), postTrialLicenseResponse.license().type());
-        });
+        assertBusy(() -> assertEquals(type.getTypeName(), getLicense().license().type()));
 
-        GetTrialStatusResponse response4 = licensingClient.prepareGetStartTrial().get();
-        assertFalse(response4.isEligibleToStartTrial());
+        assertFalse(getTrialStatus().isEligibleToStartTrial());
 
         License.LicenseType secondAttemptType = randomFrom(LicenseSettings.VALID_TRIAL_TYPES);
-
-        PostStartTrialRequestBuilder builder3 = licensingClient.preparePostStartTrial();
-        builder3.setAcknowledge(true);
-        builder3.request().setType(secondAttemptType.getTypeName());
-        PostStartTrialResponse response5 = builder3.get();
+        PostStartTrialResponse response5 = startTrial(pstr -> pstr.setType(secondAttemptType.getTypeName()).acknowledge(true));
         assertEquals(403, response5.getStatus().getRestStatus().getStatus());
         assertFalse(response5.getStatus().isTrialStarted());
         assertEquals("Operation failed: Trial was already activated.", response5.getStatus().getErrorMessage());
     }
 
     public void testInvalidType() throws Exception {
-        LicensingClient licensingClient = new LicensingClient(client());
         ensureStartingWithBasic();
-
-        PostStartTrialRequestBuilder builder = licensingClient.preparePostStartTrial();
-        builder.request().setType("basic");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::get);
+        final var future = startTrialFuture(pstr -> pstr.setType("basic").acknowledge(randomBoolean()));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, future::actionGet);
         assertThat(e.getMessage(), containsString("Cannot start trial of type [basic]. Valid trial types are ["));
     }
 
     private void ensureStartingWithBasic() throws Exception {
-        LicensingClient licensingClient = new LicensingClient(client());
-        GetLicenseResponse getLicenseResponse = licensingClient.prepareGetLicense().get();
-
-        if ("basic".equals(getLicenseResponse.license().type()) == false) {
-            licensingClient.preparePostStartBasic().setAcknowledge(true).get();
+        if ("basic".equals(getLicense().license().type()) == false) {
+            assertAcked(startBasic());
         }
 
-        assertBusy(() -> {
-            GetLicenseResponse postTrialLicenseResponse = licensingClient.prepareGetLicense().get();
-            assertEquals("basic", postTrialLicenseResponse.license().type());
-        });
+        assertBusy(() -> assertEquals("basic", getLicense().license().type()));
+    }
+
+    private ActionFuture<PostStartTrialResponse> startTrialFuture(UnaryOperator<PostStartTrialRequest> onRequest) {
+        return client().execute(PostStartTrialAction.INSTANCE, onRequest.apply(new PostStartTrialRequest(TEST_REQUEST_TIMEOUT)));
+    }
+
+    private PostStartTrialResponse startTrial(UnaryOperator<PostStartTrialRequest> onRequest) {
+        return safeGet(startTrialFuture(onRequest));
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java
index caf554015382d..14ec60dc931d3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/TestUtils.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.license.licensor.LicenseSigner;
 import org.elasticsearch.protocol.xpack.license.LicensesStatus;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -381,7 +382,9 @@ public static void registerAndAckSignedLicenses(
         License license,
         final LicensesStatus expectedStatus
     ) {
-        PutLicenseRequest putLicenseRequest = new PutLicenseRequest().license(license).acknowledge(true);
+        PutLicenseRequest putLicenseRequest = new PutLicenseRequest(ESTestCase.TEST_REQUEST_TIMEOUT, ESTestCase.TEST_REQUEST_TIMEOUT)
+            .license(license)
+            .acknowledge(true);
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<LicensesStatus> status = new AtomicReference<>();
         licenseService.registerLicense(putLicenseRequest, new ActionListener<PutLicenseResponse>() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java
index d4c8dfa5fd0a7..ab6e7356a6e02 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/XPackPluginTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.ClusterStateLicenseService;
 import org.elasticsearch.license.License;
@@ -211,7 +212,11 @@ class TestLicenseService implements MutableLicenseService {
         public void registerLicense(PutLicenseRequest request, ActionListener<PutLicenseResponse> listener) {}
 
         @Override
-        public void removeLicense(ActionListener<? extends AcknowledgedResponse> listener) {}
+        public void removeLicense(
+            TimeValue masterNodeTimeout,
+            TimeValue ackTimeout,
+            ActionListener<? extends AcknowledgedResponse> listener
+        ) {}
 
         @Override
         public void startBasicLicense(PostStartBasicRequest request, ActionListener<PostStartBasicResponse> listener) {}
diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java
index 206dfbe6729d6..803c7f410c41d 100644
--- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java
+++ b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java
@@ -118,7 +118,8 @@ public void createAndRestoreArchive() throws Exception {
 
         assertAcked(client().admin().indices().prepareDelete(indexName));
 
-        PostStartTrialRequest request = new PostStartTrialRequest().setType(License.LicenseType.TRIAL.getTypeName()).acknowledge(true);
+        PostStartTrialRequest request = new PostStartTrialRequest(TEST_REQUEST_TIMEOUT).setType(License.LicenseType.TRIAL.getTypeName())
+            .acknowledge(true);
         client().execute(PostStartTrialAction.INSTANCE, request).get();
     }
 }
diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java
index 0489390a21352..048e259eca977 100644
--- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java
+++ b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java
@@ -61,7 +61,9 @@ public void testFailRestoreOnInvalidLicense() throws Exception {
             client().execute(TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
                 .get()
         );
-        assertAcked(client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest()).get());
+        assertAcked(
+            client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)).get()
+        );
 
         ensureClusterSizeConsistency();
         ensureClusterStateConsistency();
@@ -100,7 +102,9 @@ public void testShardAllocationOnInvalidLicense() throws Exception {
             client().execute(TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
                 .get()
         );
-        assertAcked(client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest()).get());
+        assertAcked(
+            client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)).get()
+        );
 
         ensureClusterSizeConsistency();
         ensureClusterStateConsistency();
@@ -128,7 +132,8 @@ public void testShardAllocationOnInvalidLicense() throws Exception {
         waitNoPendingTasksOnAll();
         ensureClusterStateConsistency();
 
-        PostStartTrialRequest request = new PostStartTrialRequest().setType(License.LicenseType.TRIAL.getTypeName()).acknowledge(true);
+        PostStartTrialRequest request = new PostStartTrialRequest(TEST_REQUEST_TIMEOUT).setType(License.LicenseType.TRIAL.getTypeName())
+            .acknowledge(true);
         final PostStartTrialResponse response = client().execute(PostStartTrialAction.INSTANCE, request).get();
         assertThat(
             response.getStatus(),
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java
index 3a218e4ca5e97..1216db4b17608 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java
@@ -82,7 +82,9 @@ public void createAndMountSearchableSnapshot() throws Exception {
             client().execute(TransportDeleteLicenseAction.TYPE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT))
                 .get()
         );
-        assertAcked(client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest()).get());
+        assertAcked(
+            client().execute(PostStartBasicAction.INSTANCE, new PostStartBasicRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)).get()
+        );
 
         ensureClusterSizeConsistency();
         ensureClusterStateConsistency();
@@ -163,7 +165,8 @@ public void testShardAllocationOnInvalidLicense() throws Exception {
         waitNoPendingTasksOnAll();
         ensureClusterStateConsistency();
 
-        PostStartTrialRequest request = new PostStartTrialRequest().setType(License.LicenseType.TRIAL.getTypeName()).acknowledge(true);
+        PostStartTrialRequest request = new PostStartTrialRequest(TEST_REQUEST_TIMEOUT).setType(License.LicenseType.TRIAL.getTypeName())
+            .acknowledge(true);
         final PostStartTrialResponse response = client().execute(PostStartTrialAction.INSTANCE, request).get();
         assertThat(
             response.getStatus(),

From 58cb500ba67a30878bf30559ab646641ef177001 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Wed, 29 May 2024 08:14:28 +0100
Subject: [PATCH 022/208] Fix trappy timeouts in enrich module (#109136)

Relates #107984
---
 .../api/enrich.delete_policy.json             |   6 +
 .../api/enrich.execute_policy.json            |   4 +
 .../rest-api-spec/api/enrich.get_policy.json  |   6 +
 .../rest-api-spec/api/enrich.put_policy.json  |   6 +
 .../rest-api-spec/api/enrich.stats.json       |   6 +
 .../action/DeleteEnrichPolicyAction.java      |   5 +-
 .../core/enrich/action/EnrichStatsAction.java |   4 +-
 .../action/ExecuteEnrichPolicyAction.java     |   5 +-
 .../enrich/action/GetEnrichPolicyAction.java  |  16 +-
 .../enrich/action/PutEnrichPolicyAction.java  |   9 +-
 .../xpack/enrich/EnrichMultiNodeIT.java       |  36 +++--
 .../xpack/enrich/EnrichProcessorIT.java       |   6 +-
 .../xpack/enrich/EnrichRestartIT.java         |   8 +-
 .../xpack/enrich/EnrichPolicyExecutor.java    |   2 +-
 .../action/InternalExecutePolicyAction.java   |   5 +-
 .../rest/RestDeleteEnrichPolicyAction.java    |   6 +-
 .../enrich/rest/RestEnrichStatsAction.java    |   6 +-
 .../rest/RestExecuteEnrichPolicyAction.java   |   6 +-
 .../rest/RestGetEnrichPolicyAction.java       |  10 +-
 .../rest/RestPutEnrichPolicyAction.java       |   3 +-
 .../xpack/enrich/BasicEnrichTests.java        |  41 +++--
 .../enrich/EnrichPolicyExecutorTests.java     |  19 ++-
 .../xpack/enrich/EnrichPolicyUpdateTests.java |   9 +-
 .../xpack/enrich/EnrichResiliencyTests.java   |   6 +-
 .../DeleteEnrichPolicyActionRequestTests.java |   2 +-
 ...ExecuteEnrichPolicyActionRequestTests.java |   2 +-
 .../GetEnrichPolicyActionRequestTests.java    |   2 +-
 ...ternalExecutePolicyActionRequestTests.java |   4 +-
 .../PutEnrichPolicyActionRequestTests.java    |   2 +-
 ...ransportDeleteEnrichPolicyActionTests.java | 144 +++++++++++-------
 .../TransportGetEnrichPolicyActionTests.java  |  55 ++++---
 .../esql/action/CrossClustersEnrichIT.java    |  17 ++-
 .../xpack/esql/action/EnrichIT.java           |   8 +-
 .../enrich/EnrichStatsCollector.java          |   2 +-
 34 files changed, 280 insertions(+), 188 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.delete_policy.json b/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.delete_policy.json
index 3137f6b555361..5c6b05a548987 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.delete_policy.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.delete_policy.json
@@ -22,6 +22,12 @@
           }
         }
       ]
+    },
+    "params": {
+      "master_timeout":{
+        "type":"time",
+        "description":"Timeout for processing on master node"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.execute_policy.json b/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.execute_policy.json
index 5e4c8a2251d1d..2add255148508 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.execute_policy.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.execute_policy.json
@@ -28,6 +28,10 @@
         "type":"boolean",
         "default":true,
         "description":"Should the request should block until the execution is complete."
+      },
+      "master_timeout":{
+        "type":"time",
+        "description":"Timeout for processing on master node"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.get_policy.json b/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.get_policy.json
index a3eb51942c4fa..aed7397877393 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.get_policy.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.get_policy.json
@@ -26,6 +26,12 @@
           "methods": [ "GET" ]
         }
       ]
+    },
+    "params": {
+      "master_timeout":{
+        "type":"time",
+        "description":"Timeout for processing on master node"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.put_policy.json b/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.put_policy.json
index 0d1cefd3e40aa..287c7d96dca9d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.put_policy.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.put_policy.json
@@ -27,6 +27,12 @@
     "body": {
       "description": "The enrich policy to register",
       "required": true
+    },
+    "params": {
+      "master_timeout":{
+        "type":"time",
+        "description":"Timeout for processing on master node"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.stats.json
index b4218acf30eac..afd314a0dc804 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/enrich.stats.json
@@ -16,6 +16,12 @@
           "methods": [ "GET" ]
         }
       ]
+    },
+    "params": {
+      "master_timeout":{
+        "type":"time",
+        "description":"Timeout for processing on master node"
+      }
     }
   }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/DeleteEnrichPolicyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/DeleteEnrichPolicyAction.java
index 82f98176838ee..64dcd5afcb544 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/DeleteEnrichPolicyAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/DeleteEnrichPolicyAction.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -29,8 +30,8 @@ public static class Request extends MasterNodeRequest<DeleteEnrichPolicyAction.R
 
         private final String name;
 
-        public Request(String name) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String name) {
+            super(masterNodeTimeout);
             this.name = Objects.requireNonNull(name, "name cannot be null");
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java
index 3d34689c6cd0e..46425a526c53f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java
@@ -35,8 +35,8 @@ private EnrichStatsAction() {
 
     public static class Request extends MasterNodeRequest<Request> {
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout) {
+            super(masterNodeTimeout);
         }
 
         public Request(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/ExecuteEnrichPolicyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/ExecuteEnrichPolicyAction.java
index 5d629365a8096..65bae5e94852c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/ExecuteEnrichPolicyAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/ExecuteEnrichPolicyAction.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -33,8 +34,8 @@ public static class Request extends MasterNodeRequest<Request> {
         private final String name;
         private boolean waitForCompletion;
 
-        public Request(String name) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String name) {
+            super(masterNodeTimeout);
             this.name = Objects.requireNonNull(name, "name cannot be null");
             this.waitForCompletion = true;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/GetEnrichPolicyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/GetEnrichPolicyAction.java
index 37851a3641ebd..7f138dec7ee23 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/GetEnrichPolicyAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/GetEnrichPolicyAction.java
@@ -12,13 +12,12 @@
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -38,19 +37,14 @@ public static class Request extends MasterNodeReadRequest<Request> {
 
         private final List<String> names;
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
-            this.names = new ArrayList<>();
-        }
-
-        public Request(String[] names) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
-            this.names = Arrays.asList(names);
+        public Request(TimeValue masterNodeTimeout, String... names) {
+            super(masterNodeTimeout);
+            this.names = List.of(names);
         }
 
         public Request(StreamInput in) throws IOException {
             super(in);
-            this.names = in.readStringCollectionAsList();
+            this.names = in.readStringCollectionAsImmutableList();
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java
index d1031828e0522..6a6b6ff34d60d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/PutEnrichPolicyAction.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
 
@@ -27,8 +28,8 @@ private PutEnrichPolicyAction() {
         super(NAME);
     }
 
-    public static Request fromXContent(XContentParser parser, String name) throws IOException {
-        return new Request(name, EnrichPolicy.fromXContent(parser));
+    public static Request fromXContent(TimeValue masterNodeTimeout, XContentParser parser, String name) throws IOException {
+        return new Request(masterNodeTimeout, name, EnrichPolicy.fromXContent(parser));
     }
 
     public static class Request extends MasterNodeRequest<PutEnrichPolicyAction.Request> {
@@ -36,8 +37,8 @@ public static class Request extends MasterNodeRequest<PutEnrichPolicyAction.Requ
         private final EnrichPolicy policy;
         private final String name;
 
-        public Request(String name, EnrichPolicy policy) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String name, EnrichPolicy policy) {
+            super(masterNodeTimeout);
             this.name = Objects.requireNonNull(name, "name cannot be null");
             this.policy = policy;
         }
diff --git a/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java b/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java
index 1960913a6d4f4..62a5098f7a1e4 100644
--- a/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java
+++ b/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java
@@ -102,13 +102,14 @@ public void testEnrichAPIs() throws ExecutionException, InterruptedException {
                 MATCH_FIELD,
                 List.of(DECORATE_FIELDS)
             );
-            PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy);
+            PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName, enrichPolicy);
             client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet();
-            client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet();
+            client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName))
+                .actionGet();
 
             EnrichPolicy.NamedPolicy result = client().execute(
                 GetEnrichPolicyAction.INSTANCE,
-                new GetEnrichPolicyAction.Request(new String[] { policyName })
+                new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName)
             ).actionGet().getPolicies().get(0);
             assertThat(result, equalTo(new EnrichPolicy.NamedPolicy(policyName, enrichPolicy)));
             String enrichIndexPrefix = EnrichPolicy.getBaseName(policyName) + "*";
@@ -116,16 +117,19 @@ public void testEnrichAPIs() throws ExecutionException, InterruptedException {
             assertHitCount(client().search(new SearchRequest(enrichIndexPrefix)), numDocsInSourceIndex);
         }
 
-        GetEnrichPolicyAction.Response response = client().execute(GetEnrichPolicyAction.INSTANCE, new GetEnrichPolicyAction.Request())
-            .actionGet();
+        GetEnrichPolicyAction.Response response = client().execute(
+            GetEnrichPolicyAction.INSTANCE,
+            new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT)
+        ).actionGet();
         assertThat(response.getPolicies().size(), equalTo(numPolicies));
 
         for (int i = 0; i < numPolicies; i++) {
             String policyName = POLICY_NAME + i;
-            client().execute(DeleteEnrichPolicyAction.INSTANCE, new DeleteEnrichPolicyAction.Request(policyName)).actionGet();
+            client().execute(DeleteEnrichPolicyAction.INSTANCE, new DeleteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName))
+                .actionGet();
         }
 
-        response = client().execute(GetEnrichPolicyAction.INSTANCE, new GetEnrichPolicyAction.Request()).actionGet();
+        response = client().execute(GetEnrichPolicyAction.INSTANCE, new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT)).actionGet();
         assertThat(response.getPolicies().size(), equalTo(0));
     }
 
@@ -188,9 +192,9 @@ public void testExecutePolicyWithDedicatedMasterNodes() throws Exception {
             MATCH_FIELD,
             List.of(DECORATE_FIELDS)
         );
-        var putPolicyRequest = new PutEnrichPolicyAction.Request(POLICY_NAME, enrichPolicy);
+        var putPolicyRequest = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, POLICY_NAME, enrichPolicy);
         assertAcked(client().execute(PutEnrichPolicyAction.INSTANCE, putPolicyRequest).actionGet());
-        var executePolicyRequest = new ExecuteEnrichPolicyAction.Request(POLICY_NAME);
+        var executePolicyRequest = new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, POLICY_NAME);
         executePolicyRequest.setWaitForCompletion(false); // From tne returned taks id the node that executes the policy can be determined
         var executePolicyResponse = client().execute(ExecuteEnrichPolicyAction.INSTANCE, executePolicyRequest).actionGet();
         assertThat(executePolicyResponse.getStatus(), nullValue());
@@ -215,9 +219,9 @@ public void testExecutePolicyNeverOnElectedMaster() throws Exception {
             MATCH_FIELD,
             List.of(DECORATE_FIELDS)
         );
-        var putPolicyRequest = new PutEnrichPolicyAction.Request(POLICY_NAME, enrichPolicy);
+        var putPolicyRequest = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, POLICY_NAME, enrichPolicy);
         assertAcked(client().execute(PutEnrichPolicyAction.INSTANCE, putPolicyRequest).actionGet());
-        var executePolicyRequest = new ExecuteEnrichPolicyAction.Request(POLICY_NAME);
+        var executePolicyRequest = new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, POLICY_NAME);
         executePolicyRequest.setWaitForCompletion(false); // From tne returned taks id the node that executes the policy can be determined
         var executePolicyResponse = client().execute(ExecuteEnrichPolicyAction.INSTANCE, executePolicyRequest).actionGet();
         assertThat(executePolicyResponse.getStatus(), nullValue());
@@ -264,8 +268,10 @@ private static void enrich(Map<String, List<String>> keys, String coordinatingNo
             }
         }
 
-        EnrichStatsAction.Response statsResponse = client().execute(EnrichStatsAction.INSTANCE, new EnrichStatsAction.Request())
-            .actionGet();
+        EnrichStatsAction.Response statsResponse = client().execute(
+            EnrichStatsAction.INSTANCE,
+            new EnrichStatsAction.Request(TEST_REQUEST_TIMEOUT)
+        ).actionGet();
         assertThat(statsResponse.getCoordinatorStats().size(), equalTo(internalCluster().size()));
         String nodeId = getNodeId(coordinatingNode);
         CoordinatorStats stats = statsResponse.getCoordinatorStats().stream().filter(s -> s.nodeId().equals(nodeId)).findAny().get();
@@ -321,11 +327,11 @@ private static void createAndExecutePolicy(String policyName, String indexName)
             MATCH_FIELD,
             List.of(DECORATE_FIELDS)
         );
-        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy);
+        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName, enrichPolicy);
         client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet();
         final ActionFuture<ExecuteEnrichPolicyAction.Response> policyExecuteFuture = client().execute(
             ExecuteEnrichPolicyAction.INSTANCE,
-            new ExecuteEnrichPolicyAction.Request(policyName)
+            new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName)
         );
         // Make sure we can deserialize enrich policy execution task status
         final List<TaskInfo> tasks = clusterAdmin().prepareListTasks().setActions(EnrichPolicyExecutor.TASK_ACTION).get().getTasks();
diff --git a/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichProcessorIT.java b/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichProcessorIT.java
index f3d2403ce5d96..d646aed11d7d9 100644
--- a/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichProcessorIT.java
+++ b/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichProcessorIT.java
@@ -49,7 +49,7 @@ protected Settings nodeSettings() {
 
     public void testEnrichCacheValuesCannotBeCorrupted() {
         // Ensure enrich cache is empty
-        var statsRequest = new EnrichStatsAction.Request();
+        var statsRequest = new EnrichStatsAction.Request(TEST_REQUEST_TIMEOUT);
         var statsResponse = client().execute(EnrichStatsAction.INSTANCE, statsRequest).actionGet();
         assertThat(statsResponse.getCacheStats().size(), equalTo(1));
         assertThat(statsResponse.getCacheStats().get(0).count(), equalTo(0L));
@@ -85,9 +85,9 @@ public void testEnrichCacheValuesCannotBeCorrupted() {
         client().index(indexRequest).actionGet();
 
         // Store policy and execute it:
-        var putPolicyRequest = new PutEnrichPolicyAction.Request(policyName, enrichPolicy);
+        var putPolicyRequest = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName, enrichPolicy);
         client().execute(PutEnrichPolicyAction.INSTANCE, putPolicyRequest).actionGet();
-        var executePolicyRequest = new ExecuteEnrichPolicyAction.Request(policyName);
+        var executePolicyRequest = new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName);
         client().execute(ExecuteEnrichPolicyAction.INSTANCE, executePolicyRequest).actionGet();
 
         var simulatePipelineRequest = new SimulatePipelineRequest(new BytesArray("""
diff --git a/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java b/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java
index 86d18bcbbbbc4..9a77bea4ab78a 100644
--- a/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java
+++ b/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichRestartIT.java
@@ -60,7 +60,7 @@ public void testRestart() throws Exception {
         createSourceIndices(client(), enrichPolicy);
         for (int i = 0; i < numPolicies; i++) {
             String policyName = POLICY_NAME + i;
-            PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy);
+            PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName, enrichPolicy);
             client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet();
         }
 
@@ -71,8 +71,10 @@ public void testRestart() throws Exception {
     }
 
     private static void verifyPolicies(int numPolicies, EnrichPolicy enrichPolicy) {
-        GetEnrichPolicyAction.Response response = client().execute(GetEnrichPolicyAction.INSTANCE, new GetEnrichPolicyAction.Request())
-            .actionGet();
+        GetEnrichPolicyAction.Response response = client().execute(
+            GetEnrichPolicyAction.INSTANCE,
+            new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT)
+        ).actionGet();
         assertThat(response.getPolicies(), hasSize(numPolicies));
         for (int i = 0; i < numPolicies; i++) {
             String policyName = POLICY_NAME + i;
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java
index ecb03615307f9..2ebe268cc788d 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java
@@ -85,7 +85,7 @@ public void coordinatePolicyExecution(
         String enrichIndexName = EnrichPolicy.getIndexName(request.getName(), nowTimestamp);
         Releasable policyLock = tryLockingPolicy(request.getName(), enrichIndexName);
         try {
-            Request internalRequest = new Request(request.getName(), enrichIndexName);
+            Request internalRequest = new Request(request.masterNodeTimeout(), request.getName(), enrichIndexName);
             internalRequest.setWaitForCompletion(request.isWaitForCompletion());
             internalRequest.setParentTask(request.getParentTask());
             client.execute(InternalExecutePolicyAction.INSTANCE, internalRequest, ActionListener.wrap(response -> {
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java
index 769a86c5ec5b1..ed28599da9fbb 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyAction.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskAwareRequest;
 import org.elasticsearch.tasks.TaskCancelledException;
@@ -68,8 +69,8 @@ public static class Request extends ExecuteEnrichPolicyAction.Request {
 
         private final String enrichIndexName;
 
-        public Request(String name, String enrichIndexName) {
-            super(name);
+        public Request(TimeValue masterNodeTimeout, String name, String enrichIndexName) {
+            super(masterNodeTimeout, name);
             this.enrichIndexName = enrichIndexName;
         }
 
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestDeleteEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestDeleteEnrichPolicyAction.java
index 26597f86b833c..810ec48edee8e 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestDeleteEnrichPolicyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestDeleteEnrichPolicyAction.java
@@ -9,12 +9,12 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.enrich.action.DeleteEnrichPolicyAction;
 
-import java.io.IOException;
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
@@ -33,8 +33,8 @@ public String getName() {
     }
 
     @Override
-    protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) throws IOException {
-        final DeleteEnrichPolicyAction.Request request = new DeleteEnrichPolicyAction.Request(restRequest.param("name"));
+    protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
+        final var request = new DeleteEnrichPolicyAction.Request(RestUtils.getMasterNodeTimeout(restRequest), restRequest.param("name"));
         return channel -> client.execute(DeleteEnrichPolicyAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestEnrichStatsAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestEnrichStatsAction.java
index e666319b563ea..3d64e7c1380fe 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestEnrichStatsAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestEnrichStatsAction.java
@@ -9,12 +9,12 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.enrich.action.EnrichStatsAction;
 
-import java.io.IOException;
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -33,8 +33,8 @@ public String getName() {
     }
 
     @Override
-    protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) throws IOException {
-        final EnrichStatsAction.Request request = new EnrichStatsAction.Request();
+    protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
+        final var request = new EnrichStatsAction.Request(RestUtils.getMasterNodeTimeout(restRequest));
         return channel -> client.execute(EnrichStatsAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestExecuteEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestExecuteEnrichPolicyAction.java
index 15f5bdb736621..523e0bf25a71f 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestExecuteEnrichPolicyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestExecuteEnrichPolicyAction.java
@@ -9,12 +9,12 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction;
 
-import java.io.IOException;
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
@@ -34,8 +34,8 @@ public String getName() {
     }
 
     @Override
-    protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) throws IOException {
-        final ExecuteEnrichPolicyAction.Request request = new ExecuteEnrichPolicyAction.Request(restRequest.param("name"));
+    protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
+        final var request = new ExecuteEnrichPolicyAction.Request(RestUtils.getMasterNodeTimeout(restRequest), restRequest.param("name"));
         request.setWaitForCompletion(restRequest.paramAsBoolean("wait_for_completion", true));
         return channel -> client.execute(ExecuteEnrichPolicyAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestGetEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestGetEnrichPolicyAction.java
index 79dcd9315652f..2fb9f63c1eb4a 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestGetEnrichPolicyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestGetEnrichPolicyAction.java
@@ -10,12 +10,12 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.enrich.action.GetEnrichPolicyAction;
 
-import java.io.IOException;
 import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -34,9 +34,11 @@ public String getName() {
     }
 
     @Override
-    protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) throws IOException {
-        String[] names = Strings.splitStringByCommaToArray(restRequest.param("name"));
-        final GetEnrichPolicyAction.Request request = new GetEnrichPolicyAction.Request(names);
+    protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
+        final var request = new GetEnrichPolicyAction.Request(
+            RestUtils.getMasterNodeTimeout(restRequest),
+            Strings.splitStringByCommaToArray(restRequest.param("name"))
+        );
         return channel -> client.execute(GetEnrichPolicyAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestPutEnrichPolicyAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestPutEnrichPolicyAction.java
index fb1522441fe43..f172d2e0cf411 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestPutEnrichPolicyAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/rest/RestPutEnrichPolicyAction.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -41,7 +42,7 @@ protected RestChannelConsumer prepareRequest(final RestRequest restRequest, fina
 
     static PutEnrichPolicyAction.Request createRequest(RestRequest restRequest) throws IOException {
         try (XContentParser parser = restRequest.contentOrSourceParamParser()) {
-            return PutEnrichPolicyAction.fromXContent(parser, restRequest.param("name"));
+            return PutEnrichPolicyAction.fromXContent(RestUtils.getMasterNodeTimeout(restRequest), parser, restRequest.param("name"));
         }
     }
 }
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java
index e3822b366e122..d17728fdd8037 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java
@@ -93,9 +93,10 @@ public void testIngestDataWithMatchProcessor() {
             MATCH_FIELD,
             List.of(DECORATE_FIELDS)
         );
-        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy);
+        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName, enrichPolicy);
         client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet();
-        client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet();
+        client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName))
+            .actionGet();
 
         String pipelineName = "my-pipeline";
         String pipelineBody = Strings.format("""
@@ -146,8 +147,10 @@ public void testIngestDataWithMatchProcessor() {
             }
         }
 
-        EnrichStatsAction.Response statsResponse = client().execute(EnrichStatsAction.INSTANCE, new EnrichStatsAction.Request())
-            .actionGet();
+        EnrichStatsAction.Response statsResponse = client().execute(
+            EnrichStatsAction.INSTANCE,
+            new EnrichStatsAction.Request(TEST_REQUEST_TIMEOUT)
+        ).actionGet();
         assertThat(statsResponse.getCoordinatorStats().size(), equalTo(1));
         String localNodeId = getInstanceFromNode(ClusterService.class).localNode().getId();
         assertThat(statsResponse.getCoordinatorStats().get(0).nodeId(), equalTo(localNodeId));
@@ -186,9 +189,10 @@ public void testIngestDataWithGeoMatchProcessor() {
             matchField,
             List.of(enrichField)
         );
-        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy);
+        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName, enrichPolicy);
         client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet();
-        client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet();
+        client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName))
+            .actionGet();
 
         String pipelineName = "my-pipeline";
         String pipelineBody = Strings.format("""
@@ -226,8 +230,10 @@ public void testIngestDataWithGeoMatchProcessor() {
         assertThat(entries.containsKey(matchField), is(true));
         assertThat(entries.get(enrichField), equalTo("94040"));
 
-        EnrichStatsAction.Response statsResponse = client().execute(EnrichStatsAction.INSTANCE, new EnrichStatsAction.Request())
-            .actionGet();
+        EnrichStatsAction.Response statsResponse = client().execute(
+            EnrichStatsAction.INSTANCE,
+            new EnrichStatsAction.Request(TEST_REQUEST_TIMEOUT)
+        ).actionGet();
         assertThat(statsResponse.getCoordinatorStats().size(), equalTo(1));
         String localNodeId = getInstanceFromNode(ClusterService.class).localNode().getId();
         assertThat(statsResponse.getCoordinatorStats().get(0).nodeId(), equalTo(localNodeId));
@@ -246,9 +252,10 @@ public void testMultiplePolicies() {
             client().admin().indices().refresh(new RefreshRequest("source-" + i)).actionGet();
 
             EnrichPolicy enrichPolicy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of("source-" + i), "key", List.of("value"));
-            PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy);
+            PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName, enrichPolicy);
             client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet();
-            client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet();
+            client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName))
+                .actionGet();
 
             String pipelineName = "pipeline" + i;
             String pipelineBody = Strings.format("""
@@ -290,11 +297,11 @@ public void testAsyncTaskExecute() throws Exception {
         }
 
         EnrichPolicy enrichPolicy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of(sourceIndexName), "key", List.of("value"));
-        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy);
+        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName, enrichPolicy);
         client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet();
         ExecuteEnrichPolicyAction.Response executeResponse = client().execute(
             ExecuteEnrichPolicyAction.INSTANCE,
-            new ExecuteEnrichPolicyAction.Request(policyName).setWaitForCompletion(false)
+            new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName).setWaitForCompletion(false)
         ).actionGet();
 
         assertThat(executeResponse.getStatus(), is(nullValue()));
@@ -346,9 +353,10 @@ public void testTemplating() throws Exception {
             MATCH_FIELD,
             List.of(DECORATE_FIELDS)
         );
-        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy);
+        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName, enrichPolicy);
         client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet();
-        client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet();
+        client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName))
+            .actionGet();
 
         String pipelineName = "my-pipeline";
         String pipelineBody = Strings.format(
@@ -384,9 +392,10 @@ public void testFailureAfterEnrich() throws Exception {
             MATCH_FIELD,
             Arrays.asList(DECORATE_FIELDS)
         );
-        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy);
+        PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName, enrichPolicy);
         client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet();
-        client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet();
+        client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policyName))
+            .actionGet();
 
         // A pipeline with a foreach that uses a non existing field that is specified after enrich has run:
         String pipelineName = "my-pipeline";
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutorTests.java
index 9f0b18679666b..06f9eb21fe2dc 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutorTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutorTests.java
@@ -88,7 +88,7 @@ public void testNonConcurrentPolicyCoordination() throws InterruptedException {
         // Launch a fake policy run that will block until firstTaskBlock is counted down.
         final CountDownLatch firstTaskComplete = new CountDownLatch(1);
         testExecutor.coordinatePolicyExecution(
-            new ExecuteEnrichPolicyAction.Request(testPolicyName),
+            new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, testPolicyName),
             new LatchedActionListener<>(ActionListener.noop(), firstTaskComplete)
         );
 
@@ -97,7 +97,10 @@ public void testNonConcurrentPolicyCoordination() throws InterruptedException {
             EsRejectedExecutionException.class,
             "Expected exception but nothing was thrown",
             () -> {
-                testExecutor.coordinatePolicyExecution(new ExecuteEnrichPolicyAction.Request(testPolicyName), ActionListener.noop());
+                testExecutor.coordinatePolicyExecution(
+                    new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, testPolicyName),
+                    ActionListener.noop()
+                );
                 // Should throw exception on the previous statement, but if it doesn't, be a
                 // good citizen and conclude the fake runs to keep the logs clean from interrupted exceptions
                 latch.countDown();
@@ -118,7 +121,7 @@ public void testNonConcurrentPolicyCoordination() throws InterruptedException {
         // Ensure that the lock from the previous run has been cleared
         CountDownLatch secondTaskComplete = new CountDownLatch(1);
         testExecutor.coordinatePolicyExecution(
-            new ExecuteEnrichPolicyAction.Request(testPolicyName),
+            new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, testPolicyName),
             new LatchedActionListener<>(ActionListener.noop(), secondTaskComplete)
         );
         secondTaskComplete.await();
@@ -144,13 +147,13 @@ public void testMaximumPolicyExecutionLimit() throws InterruptedException {
         // Launch a two fake policy runs that will block until counted down to use up the maximum concurrent
         final CountDownLatch firstTaskComplete = new CountDownLatch(1);
         testExecutor.coordinatePolicyExecution(
-            new ExecuteEnrichPolicyAction.Request(testPolicyBaseName + "1"),
+            new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, testPolicyBaseName + "1"),
             new LatchedActionListener<>(ActionListener.noop(), firstTaskComplete)
         );
 
         final CountDownLatch secondTaskComplete = new CountDownLatch(1);
         testExecutor.coordinatePolicyExecution(
-            new ExecuteEnrichPolicyAction.Request(testPolicyBaseName + "2"),
+            new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, testPolicyBaseName + "2"),
             new LatchedActionListener<>(ActionListener.noop(), secondTaskComplete)
         );
 
@@ -160,7 +163,7 @@ public void testMaximumPolicyExecutionLimit() throws InterruptedException {
             "Expected exception but nothing was thrown",
             () -> {
                 testExecutor.coordinatePolicyExecution(
-                    new ExecuteEnrichPolicyAction.Request(testPolicyBaseName + "3"),
+                    new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, testPolicyBaseName + "3"),
                     ActionListener.noop()
                 );
                 // Should throw exception on the previous statement, but if it doesn't, be a
@@ -188,7 +191,7 @@ public void testMaximumPolicyExecutionLimit() throws InterruptedException {
         assertThat(locks.lockedPolices(), is(empty()));
         CountDownLatch finalTaskComplete = new CountDownLatch(1);
         testExecutor.coordinatePolicyExecution(
-            new ExecuteEnrichPolicyAction.Request(testPolicyBaseName + "1"),
+            new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, testPolicyBaseName + "1"),
             new LatchedActionListener<>(ActionListener.noop(), finalTaskComplete)
         );
         finalTaskComplete.await();
@@ -279,7 +282,7 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
         // Launch a fake policy run that will block until firstTaskBlock is counted down.
         PlainActionFuture<ExecuteEnrichPolicyAction.Response> firstTaskResult = new PlainActionFuture<>();
         testExecutor.coordinatePolicyExecution(
-            new ExecuteEnrichPolicyAction.Request(testPolicyName).setWaitForCompletion(false),
+            new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, testPolicyName).setWaitForCompletion(false),
             firstTaskResult
         );
 
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyUpdateTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyUpdateTests.java
index 1e4426661e06c..b015e97909179 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyUpdateTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyUpdateTests.java
@@ -52,11 +52,11 @@ public void testUpdatePolicyOnly() {
 
         EnrichPolicy instance1 = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of("index"), "key1", List.of("field1"));
         createSourceIndices(client(), instance1);
-        PutEnrichPolicyAction.Request putPolicyRequest = new PutEnrichPolicyAction.Request("my_policy", instance1);
+        PutEnrichPolicyAction.Request putPolicyRequest = new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "my_policy", instance1);
         assertAcked(client().execute(PutEnrichPolicyAction.INSTANCE, putPolicyRequest).actionGet());
         assertThat(
             "Execute failed",
-            client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request("my_policy"))
+            client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "my_policy"))
                 .actionGet()
                 .getStatus()
                 .isCompleted(),
@@ -74,7 +74,10 @@ public void testUpdatePolicyOnly() {
         createSourceIndices(client(), instance2);
         ResourceAlreadyExistsException exc = expectThrows(
             ResourceAlreadyExistsException.class,
-            client().execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request("my_policy", instance2))
+            client().execute(
+                PutEnrichPolicyAction.INSTANCE,
+                new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "my_policy", instance2)
+            )
         );
         assertTrue(exc.getMessage().contains("policy [my_policy] already exists"));
     }
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java
index 6c62d7a315872..3a2bfd87cff14 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java
@@ -80,6 +80,7 @@ public void testWriteThreadLivenessBackToBack() throws Exception {
         client().execute(
             PutEnrichPolicyAction.INSTANCE,
             new PutEnrichPolicyAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 enrichPolicyName,
                 new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of(enrichIndexName), "my_key", List.of("my_value"))
             )
@@ -87,7 +88,7 @@ public void testWriteThreadLivenessBackToBack() throws Exception {
 
         client().execute(
             ExecuteEnrichPolicyAction.INSTANCE,
-            new ExecuteEnrichPolicyAction.Request(enrichPolicyName).setWaitForCompletion(true)
+            new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, enrichPolicyName).setWaitForCompletion(true)
         ).actionGet();
 
         XContentBuilder pipe1 = JsonXContent.contentBuilder();
@@ -179,6 +180,7 @@ public void testWriteThreadLivenessWithPipeline() throws Exception {
         client().execute(
             PutEnrichPolicyAction.INSTANCE,
             new PutEnrichPolicyAction.Request(
+                TEST_REQUEST_TIMEOUT,
                 enrichPolicyName,
                 new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of(enrichIndexName), "my_key", List.of("my_value"))
             )
@@ -186,7 +188,7 @@ public void testWriteThreadLivenessWithPipeline() throws Exception {
 
         client().execute(
             ExecuteEnrichPolicyAction.INSTANCE,
-            new ExecuteEnrichPolicyAction.Request(enrichPolicyName).setWaitForCompletion(true)
+            new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, enrichPolicyName).setWaitForCompletion(true)
         ).actionGet();
 
         XContentBuilder pipe1 = JsonXContent.contentBuilder();
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/DeleteEnrichPolicyActionRequestTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/DeleteEnrichPolicyActionRequestTests.java
index 2e778d6b62215..e9cd348bf595e 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/DeleteEnrichPolicyActionRequestTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/DeleteEnrichPolicyActionRequestTests.java
@@ -13,7 +13,7 @@
 public class DeleteEnrichPolicyActionRequestTests extends AbstractWireSerializingTestCase<DeleteEnrichPolicyAction.Request> {
     @Override
     protected DeleteEnrichPolicyAction.Request createTestInstance() {
-        return new DeleteEnrichPolicyAction.Request(randomAlphaOfLength(4));
+        return new DeleteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, randomAlphaOfLength(4));
     }
 
     @Override
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/ExecuteEnrichPolicyActionRequestTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/ExecuteEnrichPolicyActionRequestTests.java
index a945f49ac97d2..08d156e1012cf 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/ExecuteEnrichPolicyActionRequestTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/ExecuteEnrichPolicyActionRequestTests.java
@@ -14,7 +14,7 @@ public class ExecuteEnrichPolicyActionRequestTests extends AbstractWireSerializi
 
     @Override
     protected ExecuteEnrichPolicyAction.Request createTestInstance() {
-        return new ExecuteEnrichPolicyAction.Request(randomAlphaOfLength(3));
+        return new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, randomAlphaOfLength(3));
     }
 
     @Override
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionRequestTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionRequestTests.java
index f84b72727bca3..051eadac48467 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionRequestTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionRequestTests.java
@@ -14,7 +14,7 @@ public class GetEnrichPolicyActionRequestTests extends AbstractWireSerializingTe
 
     @Override
     protected GetEnrichPolicyAction.Request createTestInstance() {
-        return new GetEnrichPolicyAction.Request(generateRandomStringArray(0, 4, false));
+        return new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, generateRandomStringArray(0, 4, false));
     }
 
     @Override
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionRequestTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionRequestTests.java
index 1a7bf20466ca1..68d0517a28404 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionRequestTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/InternalExecutePolicyActionRequestTests.java
@@ -19,7 +19,7 @@ protected Writeable.Reader<Request> instanceReader() {
 
     @Override
     protected Request createTestInstance() {
-        Request request = new Request(randomAlphaOfLength(3), randomAlphaOfLength(5));
+        Request request = new Request(TEST_REQUEST_TIMEOUT, randomAlphaOfLength(3), randomAlphaOfLength(5));
         if (randomBoolean()) {
             request.setWaitForCompletion(true);
         }
@@ -39,7 +39,7 @@ protected Request mutateInstance(Request instance) {
             default -> throw new AssertionError("Illegal randomisation branch");
         }
 
-        Request request = new Request(policyName, enrichIndexName);
+        Request request = new Request(TEST_REQUEST_TIMEOUT, policyName, enrichIndexName);
         request.setWaitForCompletion(waitForCompletion);
         return request;
     }
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/PutEnrichPolicyActionRequestTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/PutEnrichPolicyActionRequestTests.java
index 7675524435d26..c2f698c323004 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/PutEnrichPolicyActionRequestTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/PutEnrichPolicyActionRequestTests.java
@@ -19,7 +19,7 @@ public class PutEnrichPolicyActionRequestTests extends AbstractWireSerializingTe
     @Override
     protected PutEnrichPolicyAction.Request createTestInstance() {
         final EnrichPolicy policy = randomEnrichPolicy(XContentType.JSON);
-        return new PutEnrichPolicyAction.Request(randomAlphaOfLength(3), policy);
+        return new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, randomAlphaOfLength(3), policy);
     }
 
     @Override
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyActionTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyActionTests.java
index 84700308662b9..32f39b0de1ef4 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyActionTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/TransportDeleteEnrichPolicyActionTests.java
@@ -60,17 +60,22 @@ public void testDeletePolicyDoesNotExistUnlocksPolicy() throws InterruptedExcept
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<Exception> reference = new AtomicReference<>();
         final TransportDeleteEnrichPolicyAction transportAction = node().injector().getInstance(TransportDeleteEnrichPolicyAction.class);
-        ActionTestUtils.execute(transportAction, null, new DeleteEnrichPolicyAction.Request(fakeId), new ActionListener<>() {
-            @Override
-            public void onResponse(AcknowledgedResponse acknowledgedResponse) {
-                fail();
-            }
+        ActionTestUtils.execute(
+            transportAction,
+            null,
+            new DeleteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, fakeId),
+            new ActionListener<>() {
+                @Override
+                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                    fail();
+                }
 
-            public void onFailure(final Exception e) {
-                reference.set(e);
-                latch.countDown();
+                public void onFailure(final Exception e) {
+                    reference.set(e);
+                    latch.countDown();
+                }
             }
-        });
+        );
         latch.await();
         assertNotNull(reference.get());
         assertThat(reference.get(), instanceOf(ResourceNotFoundException.class));
@@ -92,17 +97,22 @@ public void testDeleteWithoutIndex() throws Exception {
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<AcknowledgedResponse> reference = new AtomicReference<>();
         final TransportDeleteEnrichPolicyAction transportAction = node().injector().getInstance(TransportDeleteEnrichPolicyAction.class);
-        ActionTestUtils.execute(transportAction, null, new DeleteEnrichPolicyAction.Request(name), new ActionListener<>() {
-            @Override
-            public void onResponse(AcknowledgedResponse acknowledgedResponse) {
-                reference.set(acknowledgedResponse);
-                latch.countDown();
-            }
+        ActionTestUtils.execute(
+            transportAction,
+            null,
+            new DeleteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, name),
+            new ActionListener<>() {
+                @Override
+                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                    reference.set(acknowledgedResponse);
+                    latch.countDown();
+                }
 
-            public void onFailure(final Exception e) {
-                fail();
+                public void onFailure(final Exception e) {
+                    fail();
+                }
             }
-        });
+        );
         latch.await();
         assertNotNull(reference.get());
         assertTrue(reference.get().isAcknowledged());
@@ -137,17 +147,22 @@ public void testDeleteIsNotLocked() throws Exception {
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<AcknowledgedResponse> reference = new AtomicReference<>();
         final TransportDeleteEnrichPolicyAction transportAction = node().injector().getInstance(TransportDeleteEnrichPolicyAction.class);
-        ActionTestUtils.execute(transportAction, null, new DeleteEnrichPolicyAction.Request(name), new ActionListener<>() {
-            @Override
-            public void onResponse(AcknowledgedResponse acknowledgedResponse) {
-                reference.set(acknowledgedResponse);
-                latch.countDown();
-            }
+        ActionTestUtils.execute(
+            transportAction,
+            null,
+            new DeleteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, name),
+            new ActionListener<>() {
+                @Override
+                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                    reference.set(acknowledgedResponse);
+                    latch.countDown();
+                }
 
-            public void onFailure(final Exception e) {
-                fail();
+                public void onFailure(final Exception e) {
+                    fail();
+                }
             }
-        });
+        );
         latch.await();
         assertNotNull(reference.get());
         assertTrue(reference.get().isAcknowledged());
@@ -188,17 +203,22 @@ public void testDeleteLocked() throws InterruptedException {
         {
             final CountDownLatch latch = new CountDownLatch(1);
             final AtomicReference<Exception> reference = new AtomicReference<>();
-            ActionTestUtils.execute(transportAction, null, new DeleteEnrichPolicyAction.Request(name), new ActionListener<>() {
-                @Override
-                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
-                    fail();
-                }
-
-                public void onFailure(final Exception e) {
-                    reference.set(e);
-                    latch.countDown();
+            ActionTestUtils.execute(
+                transportAction,
+                null,
+                new DeleteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, name),
+                new ActionListener<>() {
+                    @Override
+                    public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                        fail();
+                    }
+
+                    public void onFailure(final Exception e) {
+                        reference.set(e);
+                        latch.countDown();
+                    }
                 }
-            });
+            );
             latch.await();
             assertNotNull(reference.get());
             assertThat(reference.get(), instanceOf(EsRejectedExecutionException.class));
@@ -214,17 +234,22 @@ public void onFailure(final Exception e) {
             final CountDownLatch latch = new CountDownLatch(1);
             final AtomicReference<AcknowledgedResponse> reference = new AtomicReference<>();
 
-            ActionTestUtils.execute(transportAction, null, new DeleteEnrichPolicyAction.Request(name), new ActionListener<>() {
-                @Override
-                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
-                    reference.set(acknowledgedResponse);
-                    latch.countDown();
-                }
-
-                public void onFailure(final Exception e) {
-                    fail();
+            ActionTestUtils.execute(
+                transportAction,
+                null,
+                new DeleteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, name),
+                new ActionListener<>() {
+                    @Override
+                    public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                        reference.set(acknowledgedResponse);
+                        latch.countDown();
+                    }
+
+                    public void onFailure(final Exception e) {
+                        fail();
+                    }
                 }
-            });
+            );
             latch.await();
             assertNotNull(reference.get());
             assertTrue(reference.get().isAcknowledged());
@@ -256,17 +281,22 @@ public void testDeletePolicyPrefixes() throws InterruptedException {
             final CountDownLatch latch = new CountDownLatch(1);
             final AtomicReference<AcknowledgedResponse> reference = new AtomicReference<>();
 
-            ActionTestUtils.execute(transportAction, null, new DeleteEnrichPolicyAction.Request(name), new ActionListener<>() {
-                @Override
-                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
-                    reference.set(acknowledgedResponse);
-                    latch.countDown();
-                }
-
-                public void onFailure(final Exception e) {
-                    fail();
+            ActionTestUtils.execute(
+                transportAction,
+                null,
+                new DeleteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, name),
+                new ActionListener<>() {
+                    @Override
+                    public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                        reference.set(acknowledgedResponse);
+                        latch.countDown();
+                    }
+
+                    public void onFailure(final Exception e) {
+                        fail();
+                    }
                 }
-            });
+            );
             latch.await();
             assertNotNull(reference.get());
             assertTrue(reference.get().isAcknowledged());
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyActionTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyActionTests.java
index 1a95627e9438d..6a3c1eb2555b1 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyActionTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/TransportGetEnrichPolicyActionTests.java
@@ -34,7 +34,7 @@ public void cleanupPolicies() throws InterruptedException {
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<GetEnrichPolicyAction.Response> reference = new AtomicReference<>();
         final TransportGetEnrichPolicyAction transportAction = node().injector().getInstance(TransportGetEnrichPolicyAction.class);
-        ActionTestUtils.execute(transportAction, null, new GetEnrichPolicyAction.Request(), new ActionListener<>() {
+        ActionTestUtils.execute(transportAction, null, new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT), new ActionListener<>() {
             @Override
             public void onResponse(GetEnrichPolicyAction.Response response) {
                 reference.set(response);
@@ -74,24 +74,18 @@ public void testListPolicies() throws InterruptedException {
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<GetEnrichPolicyAction.Response> reference = new AtomicReference<>();
         final TransportGetEnrichPolicyAction transportAction = node().injector().getInstance(TransportGetEnrichPolicyAction.class);
-        ActionTestUtils.execute(
-            transportAction,
-            null,
-            // empty or null should return the same
-            randomBoolean() ? new GetEnrichPolicyAction.Request() : new GetEnrichPolicyAction.Request(new String[] {}),
-            new ActionListener<>() {
-                @Override
-                public void onResponse(GetEnrichPolicyAction.Response response) {
-                    reference.set(response);
-                    latch.countDown();
+        ActionTestUtils.execute(transportAction, null, new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT), new ActionListener<>() {
+            @Override
+            public void onResponse(GetEnrichPolicyAction.Response response) {
+                reference.set(response);
+                latch.countDown();
 
-                }
+            }
 
-                public void onFailure(final Exception e) {
-                    fail();
-                }
+            public void onFailure(final Exception e) {
+                fail();
             }
-        );
+        });
         latch.await();
         assertNotNull(reference.get());
         GetEnrichPolicyAction.Response response = reference.get();
@@ -107,7 +101,7 @@ public void testListEmptyPolicies() throws InterruptedException {
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<GetEnrichPolicyAction.Response> reference = new AtomicReference<>();
         final TransportGetEnrichPolicyAction transportAction = node().injector().getInstance(TransportGetEnrichPolicyAction.class);
-        ActionTestUtils.execute(transportAction, null, new GetEnrichPolicyAction.Request(), new ActionListener<>() {
+        ActionTestUtils.execute(transportAction, null, new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT), new ActionListener<>() {
             @Override
             public void onResponse(GetEnrichPolicyAction.Response response) {
                 reference.set(response);
@@ -141,17 +135,22 @@ public void testGetPolicy() throws InterruptedException {
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<GetEnrichPolicyAction.Response> reference = new AtomicReference<>();
         final TransportGetEnrichPolicyAction transportAction = node().injector().getInstance(TransportGetEnrichPolicyAction.class);
-        ActionTestUtils.execute(transportAction, null, new GetEnrichPolicyAction.Request(new String[] { name }), new ActionListener<>() {
-            @Override
-            public void onResponse(GetEnrichPolicyAction.Response response) {
-                reference.set(response);
-                latch.countDown();
-            }
+        ActionTestUtils.execute(
+            transportAction,
+            null,
+            new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, name),
+            new ActionListener<>() {
+                @Override
+                public void onResponse(GetEnrichPolicyAction.Response response) {
+                    reference.set(response);
+                    latch.countDown();
+                }
 
-            public void onFailure(final Exception e) {
-                fail();
+                public void onFailure(final Exception e) {
+                    fail();
+                }
             }
-        });
+        );
         latch.await();
         assertNotNull(reference.get());
         GetEnrichPolicyAction.Response response = reference.get();
@@ -186,7 +185,7 @@ public void testGetMultiplePolicies() throws InterruptedException {
         ActionTestUtils.execute(
             transportAction,
             null,
-            new GetEnrichPolicyAction.Request(new String[] { name, anotherName }),
+            new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, name, anotherName),
             new ActionListener<>() {
                 @Override
                 public void onResponse(GetEnrichPolicyAction.Response response) {
@@ -220,7 +219,7 @@ public void testGetPolicyThrowsError() throws InterruptedException {
         ActionTestUtils.execute(
             transportAction,
             null,
-            new GetEnrichPolicyAction.Request(new String[] { "non-exists" }),
+            new GetEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "non-exists"),
             new ActionListener<>() {
                 @Override
                 public void onResponse(GetEnrichPolicyAction.Response response) {
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
index 12708fb626c36..147e62f7ee3bc 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java
@@ -118,8 +118,10 @@ public void setupHostsEnrich() {
                 client.prepareIndex("hosts").setSource("ip", h.getKey(), "os", h.getValue()).get();
             }
             client.admin().indices().prepareRefresh("hosts").get();
-            client.execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request("hosts", hostPolicy)).actionGet();
-            client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request("hosts")).actionGet();
+            client.execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "hosts", hostPolicy))
+                .actionGet();
+            client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "hosts"))
+                .actionGet();
             assertAcked(client.admin().indices().prepareDelete("hosts"));
         }
     }
@@ -137,8 +139,10 @@ public void setupVendorPolicy() {
                 client.prepareIndex("vendors").setSource("os", v.getKey(), "vendor", v.getValue()).get();
             }
             client.admin().indices().prepareRefresh("vendors").get();
-            client.execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request("vendors", vendorPolicy)).actionGet();
-            client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request("vendors")).actionGet();
+            client.execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "vendors", vendorPolicy))
+                .actionGet();
+            client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "vendors"))
+                .actionGet();
             assertAcked(client.admin().indices().prepareDelete("vendors"));
         }
     }
@@ -195,7 +199,10 @@ public void wipeEnrichPolicies() {
         for (String cluster : allClusters()) {
             cluster(cluster).wipe(Set.of());
             for (String policy : List.of("hosts", "vendors")) {
-                client(cluster).execute(DeleteEnrichPolicyAction.INSTANCE, new DeleteEnrichPolicyAction.Request(policy));
+                client(cluster).execute(
+                    DeleteEnrichPolicyAction.INSTANCE,
+                    new DeleteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, policy)
+                );
             }
         }
     }
diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java
index 1298e3374665b..5806cb8ef0982 100644
--- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java
+++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java
@@ -166,15 +166,17 @@ record Song(String id, String title, String artist, double length) {
             client().prepareIndex("songs").setSource("song_id", s.id, "title", s.title, "artist", s.artist, "length", s.length).get();
         }
         client().admin().indices().prepareRefresh("songs").get();
-        client().execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request("songs", policy)).actionGet();
-        client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request("songs")).actionGet();
+        client().execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "songs", policy))
+            .actionGet();
+        client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "songs"))
+            .actionGet();
         assertAcked(client().admin().indices().prepareDelete("songs"));
     }
 
     @After
     public void cleanEnrichPolicies() {
         cluster().wipe(Set.of());
-        client().execute(DeleteEnrichPolicyAction.INSTANCE, new DeleteEnrichPolicyAction.Request("songs"));
+        client().execute(DeleteEnrichPolicyAction.INSTANCE, new DeleteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "songs"));
     }
 
     @Before
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichStatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichStatsCollector.java
index 234bc8f72a52b..b9743f022da84 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichStatsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichStatsCollector.java
@@ -52,7 +52,7 @@ protected Collection<MonitoringDoc> doCollect(MonitoringDoc.Node node, long inte
             final long timestamp = timestamp();
             final String clusterUuid = clusterUuid(clusterState);
 
-            final EnrichStatsAction.Request request = new EnrichStatsAction.Request();
+            final EnrichStatsAction.Request request = new EnrichStatsAction.Request(getCollectionTimeout());
             final EnrichStatsAction.Response response = client.execute(EnrichStatsAction.INSTANCE, request)
                 .actionGet(getCollectionTimeout());
 

From e622101ac97496281f8bf87e537e1e0b4db30b9a Mon Sep 17 00:00:00 2001
From: Artem Prigoda <artem.prigoda@elastic.co>
Date: Wed, 29 May 2024 09:15:02 +0200
Subject: [PATCH 023/208] Fix
 TasksIT#testGetTaskWaitForCompletionWithoutStoringResult (#108094)

It seems that the failure (the missed index) has always existed in the test scenario and it's supposed to be handled by TransportGetTaskAction.java. We catch IndexNotFoundException here and convert it to ResourceNotFoundException. Then we catch ResourceNotFoundException here and return a snapshot of a task as a response.

In the stack trace, getFinishedTaskFromIndex was called from getRunningTaskFromNode, not from waitedForCompletion due to a race between creating a get request and unblocking request which are sent asynchronously. I've changed the waitForCompletionTestCase test method to unblock the task only after the request started waiting for the task completion by registering a removal listener. By doing so, we make sure we test the "wait for completion" branch when task is running.

The part about the missed index seems to irrelevant, since waitedForCompletion is able to suppress the error and return a snapshot of running task which is not possible if getFinishedTaskFromIndex gets called directly from getRunningTaskFromNode.

Resolves #107823
---
 .../admin/cluster/node/tasks/TasksIT.java     | 35 ++++++++++++-------
 .../tasks/get/TransportGetTaskAction.java     | 14 ++++++--
 .../test/tasks/MockTaskManager.java           | 13 +++++++
 .../test/tasks/MockTaskManagerListener.java   |  3 ++
 4 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index 8011be1d69a04..5ea1b869f417e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.tasks.RemovedTaskListener;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskId;
@@ -599,24 +600,32 @@ private <T> void waitForCompletionTestCase(boolean storeResult, Function<TaskId,
             TEST_TASK_ACTION.name() + "[n]",
             () -> client().execute(TEST_TASK_ACTION, request)
         );
-        ActionFuture<T> waitResponseFuture;
+        var tasks = clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name()).get().getTasks();
+        assertThat(tasks, hasSize(1));
+        TaskId taskId = tasks.get(0).taskId();
+        clusterAdmin().prepareGetTask(taskId).get();
+
+        var taskManager = (MockTaskManager) internalCluster().getInstance(
+            TransportService.class,
+            clusterService().state().getNodes().resolveNode(taskId.getNodeId()).getName()
+        ).getTaskManager();
+        var listener = new MockTaskManagerListener() {
+            @Override
+            public void onRemovedTaskListenerRegistered(RemovedTaskListener removedTaskListener) {
+                // Unblock the request only after it started waiting for task completion
+                client().execute(UNBLOCK_TASK_ACTION, new TestTaskPlugin.UnblockTestTasksRequest());
+            }
+        };
+        taskManager.addListener(listener);
         try {
-            var tasks = clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name()).get().getTasks();
-            assertThat(tasks, hasSize(1));
-            var taskId = tasks.get(0).taskId();
-            clusterAdmin().prepareGetTask(taskId).get();
-
             // Spin up a request to wait for the test task to finish
-            waitResponseFuture = wait.apply(taskId);
+            // The task will be unblocked as soon as the request started waiting for task completion
+            T waitResponse = wait.apply(taskId).get();
+            validator.accept(waitResponse);
         } finally {
-            // Unblock the request so the wait for completion request can finish
-            client().execute(UNBLOCK_TASK_ACTION, new TestTaskPlugin.UnblockTestTasksRequest()).get();
+            taskManager.removeListener(listener);
         }
 
-        // Now that the task is unblocked the list response will come back
-        T waitResponse = waitResponseFuture.get();
-        validator.accept(waitResponse);
-
         TestTaskPlugin.NodesResponse response = future.get();
         assertEquals(emptyList(), response.failures());
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index c8b33e6d569d2..85de3c65c798e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -141,9 +141,17 @@ void getRunningTaskFromNode(Task thisTask, GetTaskRequest request, ActionListene
         } else {
             if (request.getWaitForCompletion()) {
                 final ListenableActionFuture<Void> future = new ListenableActionFuture<>();
-                RemovedTaskListener removedTaskListener = task -> {
-                    if (task.equals(runningTask)) {
-                        future.onResponse(null);
+                RemovedTaskListener removedTaskListener = new RemovedTaskListener() {
+                    @Override
+                    public void onRemoved(Task task) {
+                        if (task.equals(runningTask)) {
+                            future.onResponse(null);
+                        }
+                    }
+
+                    @Override
+                    public String toString() {
+                        return "Waiting for task completion " + runningTask;
                     }
                 };
                 taskManager.registerRemovedTaskListener(removedTaskListener);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
index be2c0c332e41c..599868ab7f1f9 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.RemovedTaskListener;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskAwareRequest;
 import org.elasticsearch.tasks.TaskManager;
@@ -84,4 +85,16 @@ public void addListener(MockTaskManagerListener listener) {
     public void removeListener(MockTaskManagerListener listener) {
         listeners.remove(listener);
     }
+
+    @Override
+    public void registerRemovedTaskListener(RemovedTaskListener removedTaskListener) {
+        super.registerRemovedTaskListener(removedTaskListener);
+        for (MockTaskManagerListener listener : listeners) {
+            try {
+                listener.onRemovedTaskListenerRegistered(removedTaskListener);
+            } catch (Exception e) {
+                logger.warn("failed to notify task manager listener about a registered removed task listener", e);
+            }
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java
index 4bebfae914219..1f915f26db70f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.test.tasks;
 
+import org.elasticsearch.tasks.RemovedTaskListener;
 import org.elasticsearch.tasks.Task;
 
 /**
@@ -17,4 +18,6 @@ public interface MockTaskManagerListener {
     default void onTaskRegistered(Task task) {};
 
     default void onTaskUnregistered(Task task) {};
+
+    default void onRemovedTaskListenerRegistered(RemovedTaskListener removedTaskListener) {};
 }

From d9e6f5452e1bcad67c805570adf033e37865cfdf Mon Sep 17 00:00:00 2001
From: Jedr Blaszyk <jedrazb@gmail.com>
Date: Wed, 29 May 2024 10:34:51 +0200
Subject: [PATCH 024/208] [Connector API] Fix bug with with wrong target index
 for access control sync (#109097)

---
 docs/changelog/109097.yaml                    |  6 ++++++
 .../sync_job/10_connector_sync_job_post.yml   | 21 +++++++++++++++++++
 .../application/connector/Connector.java      |  5 +++++
 .../connector/ConnectorTemplateRegistry.java  |  1 +
 .../syncjob/ConnectorSyncJobIndexService.java | 12 +++++++----
 .../ConnectorSyncJobIndexServiceTests.java    | 13 ++++++++++++
 6 files changed, 54 insertions(+), 4 deletions(-)
 create mode 100644 docs/changelog/109097.yaml

diff --git a/docs/changelog/109097.yaml b/docs/changelog/109097.yaml
new file mode 100644
index 0000000000000..a7520f4eaa9be
--- /dev/null
+++ b/docs/changelog/109097.yaml
@@ -0,0 +1,6 @@
+pr: 109097
+summary: "[Connector API] Fix bug with with wrong target index for access control\
+  \ sync"
+area: Application
+type: bug
+issues: []
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml
index e703b153dd3bb..8a384eee6bb93 100644
--- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml
@@ -275,6 +275,27 @@ setup:
   - exists: created_at
   - exists: last_seen
 
+---
+'Create access control sync job - expect prefixed connector index name':
+  - do:
+      connector.sync_job_post:
+        body:
+          id: test-connector
+          job_type: access_control
+
+  - set:  { id: id }
+
+  - match: { id: $id }
+
+  - do:
+      connector.sync_job_get:
+        connector_sync_job_id: $id
+
+  - match: { connector.id: test-connector }
+  - match: { job_type: access_control }
+  - match: { connector.index_name: .search-acl-filter-search-test }
+
+
 ---
 'Create connector sync job with non-existing connector id':
   - do:
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java
index e9361d78ad707..e9447149c7e6c 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java
@@ -34,6 +34,7 @@
 import java.util.Objects;
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.ACCESS_CONTROL_INDEX_PREFIX;
 
 /**
  * Represents a Connector in the Elasticsearch ecosystem. Connectors are used for integrating
@@ -269,6 +270,10 @@ public Connector(StreamInput in) throws IOException {
         }
     );
 
+    public String getAccessControlIndexName() {
+        return ACCESS_CONTROL_INDEX_PREFIX + this.indexName;
+    }
+
     static {
         PARSER.declareStringOrNull(optionalConstructorArg(), API_KEY_ID_FIELD);
         PARSER.declareStringOrNull(optionalConstructorArg(), API_KEY_SECRET_ID_FIELD);
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java
index e4ce4d8181fd8..41976bc6b4272 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java
@@ -46,6 +46,7 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry {
     public static final String CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN = ".elastic-connectors-sync-jobs-v1";
     public static final String CONNECTOR_SYNC_JOBS_TEMPLATE_NAME = "elastic-connectors-sync-jobs";
 
+    public static final String ACCESS_CONTROL_INDEX_PREFIX = ".search-acl-filter-";
     public static final String ACCESS_CONTROL_INDEX_NAME_PATTERN = ".search-acl-filter-*";
     public static final String ACCESS_CONTROL_TEMPLATE_NAME = "search-acl-filter";
 
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java
index 85f8ef089c6fe..6d81bae3c2e9f 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java
@@ -96,8 +96,9 @@ public void createConnectorSyncJob(
         ActionListener<PostConnectorSyncJobAction.Response> listener
     ) {
         String connectorId = request.getId();
+        ConnectorSyncJobType jobType = Objects.requireNonNullElse(request.getJobType(), ConnectorSyncJob.DEFAULT_JOB_TYPE);
         try {
-            getSyncJobConnectorInfo(connectorId, listener.delegateFailure((l, connector) -> {
+            getSyncJobConnectorInfo(connectorId, jobType, listener.delegateFailure((l, connector) -> {
                 if (Strings.isNullOrEmpty(connector.getIndexName())) {
                     l.onFailure(
                         new ElasticsearchStatusException(
@@ -125,7 +126,6 @@ public void createConnectorSyncJob(
                 }
 
                 Instant now = Instant.now();
-                ConnectorSyncJobType jobType = Objects.requireNonNullElse(request.getJobType(), ConnectorSyncJob.DEFAULT_JOB_TYPE);
                 ConnectorSyncJobTriggerMethod triggerMethod = Objects.requireNonNullElse(
                     request.getTriggerMethod(),
                     ConnectorSyncJob.DEFAULT_TRIGGER_METHOD
@@ -494,7 +494,7 @@ private ConnectorSyncStatus getConnectorSyncJobStatusFromSearchResult(ConnectorS
         );
     }
 
-    private void getSyncJobConnectorInfo(String connectorId, ActionListener<Connector> listener) {
+    private void getSyncJobConnectorInfo(String connectorId, ConnectorSyncJobType jobType, ActionListener<Connector> listener) {
         try {
 
             final GetRequest request = new GetRequest(ConnectorIndexService.CONNECTOR_INDEX_NAME, connectorId);
@@ -514,11 +514,15 @@ public void onResponse(GetResponse response) {
                             connectorId,
                             XContentType.JSON
                         );
+                        // Access control syncs write data to a separate index
+                        String targetIndexName = jobType == ConnectorSyncJobType.ACCESS_CONTROL
+                            ? connector.getAccessControlIndexName()
+                            : connector.getIndexName();
 
                         // Build the connector representation for sync job
                         final Connector syncJobConnector = new Connector.Builder().setConnectorId(connector.getConnectorId())
                             .setSyncJobFiltering(transformConnectorFilteringToSyncJobRepresentation(connector.getFiltering()))
-                            .setIndexName(connector.getIndexName())
+                            .setIndexName(targetIndexName)
                             .setLanguage(connector.getLanguage())
                             .setPipeline(connector.getPipeline())
                             .setServiceType(connector.getServiceType())
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java
index 97ebea64eb2d8..5e854bb9d2396 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java
@@ -56,6 +56,7 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.ACCESS_CONTROL_INDEX_PREFIX;
 import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.registerSimplifiedConnectorIndexTemplates;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -131,6 +132,18 @@ public void testCreateConnectorSyncJob() throws Exception {
         assertThat(connectorSyncJob.getDeletedDocumentCount(), equalTo(0L));
     }
 
+    public void testCreateConnectorSyncJob_WithAccessControlJobType_IndexIsPrefixed() throws Exception {
+        PostConnectorSyncJobAction.Request createAccessControlJobRequest = ConnectorSyncJobTestUtils
+            .getRandomPostConnectorSyncJobActionRequest(connectorOneId, ConnectorSyncJobType.ACCESS_CONTROL);
+
+        PostConnectorSyncJobAction.Response createAccessControlJobResponse = awaitPutConnectorSyncJob(createAccessControlJobRequest);
+
+        ConnectorSyncJob connectorSyncJob = awaitGetConnectorSyncJob(createAccessControlJobResponse.getId());
+
+        assertThat(connectorSyncJob.getJobType(), equalTo(ConnectorSyncJobType.ACCESS_CONTROL));
+        assertTrue(connectorSyncJob.getConnector().getIndexName().startsWith(ACCESS_CONTROL_INDEX_PREFIX));
+    }
+
     public void testCreateConnectorSyncJob_WithMissingJobType_ExpectDefaultJobTypeToBeSet() throws Exception {
         PostConnectorSyncJobAction.Request syncJobRequest = new PostConnectorSyncJobAction.Request(
             connectorOneId,

From b84ad26c9b13fe4733fb30fd11aa353fc37e1a85 Mon Sep 17 00:00:00 2001
From: Tim Brooks <tim@timbrooks.org>
Date: Wed, 29 May 2024 02:56:43 -0600
Subject: [PATCH 025/208] Use the unsafe future for read single region
 (#109139)

We need to use the unsafe future to block on the same thread pool
similar to read multi regions.
---
 .../blobcache/shared/SharedBlobCacheService.java              | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
index c5ef1d7c2bf1d..218f12a97e1a6 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
@@ -1116,7 +1116,9 @@ private int readSingleRegion(
             RangeMissingHandler writer,
             int region
         ) throws InterruptedException, ExecutionException {
-            final PlainActionFuture<Integer> readFuture = new PlainActionFuture<>();
+            final PlainActionFuture<Integer> readFuture = new UnsafePlainActionFuture<>(
+                BlobStoreRepository.STATELESS_SHARD_PREWARMING_THREAD_NAME
+            );
             final CacheFileRegion fileRegion = get(cacheKey, length, region);
             final long regionStart = getRegionStart(region);
             fileRegion.populateAndRead(

From 1044a0104f6868c52fcc4161f7f973ed3e5a4087 Mon Sep 17 00:00:00 2001
From: Jedr Blaszyk <jedrazb@gmail.com>
Date: Wed, 29 May 2024 11:48:58 +0200
Subject: [PATCH 026/208] [Connecor API] Add references to tutorial in docs
 (#109099)

---
 .../connector/apis/cancel-connector-sync-job-api.asciidoc       | 2 ++
 docs/reference/connector/apis/check-in-connector-api.asciidoc   | 2 ++
 .../connector/apis/check-in-connector-sync-job-api.asciidoc     | 2 ++
 docs/reference/connector/apis/connector-apis.asciidoc           | 2 +-
 docs/reference/connector/apis/create-connector-api.asciidoc     | 2 ++
 .../connector/apis/create-connector-sync-job-api.asciidoc       | 2 ++
 docs/reference/connector/apis/delete-connector-api.asciidoc     | 2 ++
 .../connector/apis/delete-connector-sync-job-api.asciidoc       | 2 ++
 docs/reference/connector/apis/get-connector-api.asciidoc        | 2 ++
 .../connector/apis/get-connector-sync-job-api.asciidoc          | 2 ++
 .../connector/apis/list-connector-sync-jobs-api.asciidoc        | 1 +
 docs/reference/connector/apis/list-connectors-api.asciidoc      | 2 ++
 .../connector/apis/set-connector-sync-job-error-api.asciidoc    | 2 ++
 .../connector/apis/set-connector-sync-job-stats-api.asciidoc    | 2 ++
 .../connector/apis/update-connector-api-key-id-api.asciidoc     | 2 ++
 .../connector/apis/update-connector-configuration-api.asciidoc  | 1 +
 .../connector/apis/update-connector-error-api.asciidoc          | 2 ++
 .../connector/apis/update-connector-filtering-api.asciidoc      | 2 ++
 .../connector/apis/update-connector-index-name-api.asciidoc     | 2 ++
 .../connector/apis/update-connector-last-sync-api.asciidoc      | 2 ++
 .../apis/update-connector-name-description-api.asciidoc         | 2 ++
 .../connector/apis/update-connector-pipeline-api.asciidoc       | 2 ++
 .../connector/apis/update-connector-scheduling-api.asciidoc     | 2 ++
 .../connector/apis/update-connector-service-type-api.asciidoc   | 2 ++
 .../connector/apis/update-connector-status-api.asciidoc         | 2 ++
 25 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc
index 0519cfdf15984..105d395e42c8b 100644
--- a/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc
@@ -8,6 +8,8 @@ beta::[]
 
 Cancels a connector sync job.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[cancel-connector-sync-job-api-request]]
 ==== {api-request-title}
 `PUT _connector/_sync_job/<connector_sync_job_id>/_cancel`
diff --git a/docs/reference/connector/apis/check-in-connector-api.asciidoc b/docs/reference/connector/apis/check-in-connector-api.asciidoc
index e8119028ea24d..8c6b5161a3a72 100644
--- a/docs/reference/connector/apis/check-in-connector-api.asciidoc
+++ b/docs/reference/connector/apis/check-in-connector-api.asciidoc
@@ -8,6 +8,8 @@ preview::[]
 
 Updates the `last_seen` field of a connector with current timestamp.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[check-in-connector-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc
index 482c2e8dd4a2f..a052fbb2418cc 100644
--- a/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc
@@ -8,6 +8,8 @@ preview::[]
 
 Checks in a connector sync job (updates `last_seen` to the current time).
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[check-in-connector-sync-job-api-request]]
 ==== {api-request-title}
 `PUT _connector/_sync_job/<connector_sync_job_id>/_check_in/`
diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc
index c8b06bb1d0a6e..9d23fcf8d336e 100644
--- a/docs/reference/connector/apis/connector-apis.asciidoc
+++ b/docs/reference/connector/apis/connector-apis.asciidoc
@@ -3,7 +3,7 @@
 
 beta::[]
 
-The connector and sync jobs APIs provide a convenient way to create and manage Elastic {enterprise-search-ref}/connectors.html[connectors^] and sync jobs in an internal index.
+The connector and sync jobs APIs provide a convenient way to create and manage Elastic {enterprise-search-ref}/connectors.html[connectors^] and sync jobs in an internal index. To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
 
 Connectors are {es} integrations that bring content from third-party data sources, which can be deployed on {ecloud} or hosted on your own infrastructure:
 
diff --git a/docs/reference/connector/apis/create-connector-api.asciidoc b/docs/reference/connector/apis/create-connector-api.asciidoc
index 2d7944c52f21b..9bd49a3c5ef94 100644
--- a/docs/reference/connector/apis/create-connector-api.asciidoc
+++ b/docs/reference/connector/apis/create-connector-api.asciidoc
@@ -14,6 +14,8 @@ Connectors are {es} integrations that bring content from third-party data source
 
 Find a list of all supported service types in the {enterprise-search-ref}/connectors.html[connectors documentation^].
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [source,console]
 --------------------------------------------------
 PUT _connector/my-connector
diff --git a/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc
index 43a2339c56847..c7cc866930dfb 100644
--- a/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc
@@ -9,6 +9,8 @@ beta::[]
 
 Creates a connector sync job.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [source, console]
 --------------------------------------------------
 POST _connector/_sync_job
diff --git a/docs/reference/connector/apis/delete-connector-api.asciidoc b/docs/reference/connector/apis/delete-connector-api.asciidoc
index d8dfea8c401ce..23acd1b4755b1 100644
--- a/docs/reference/connector/apis/delete-connector-api.asciidoc
+++ b/docs/reference/connector/apis/delete-connector-api.asciidoc
@@ -11,6 +11,8 @@ This is a destructive action that is not recoverable.
 
 Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[delete-connector-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc
index 0c3274454845c..7cdabb22f05ee 100644
--- a/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc
@@ -9,6 +9,8 @@ beta::[]
 Removes a connector sync job and its associated data.
 This is a destructive action that is not recoverable.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[delete-connector-sync-job-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc
index 7e5186b146c51..4df792c8a0a1a 100644
--- a/docs/reference/connector/apis/get-connector-api.asciidoc
+++ b/docs/reference/connector/apis/get-connector-api.asciidoc
@@ -8,6 +8,8 @@ beta::[]
 
 Retrieves the details about a connector.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[get-connector-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc
index e162fe191375c..fffdada2a2a82 100644
--- a/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc
+++ b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc
@@ -8,6 +8,8 @@ beta::[]
 
 Retrieves the details about a connector sync job.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[get-connector-sync-job-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc
index 6d06e7e6b9045..217b29451937d 100644
--- a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc
+++ b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc
@@ -9,6 +9,7 @@ beta::[]
 
 Returns information about all stored connector sync jobs ordered by their creation date in ascending order.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
 
 [[list-connector-sync-jobs-api-request]]
 ==== {api-request-title}
diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc
index c24acec5e82ce..c7ea2afd8102f 100644
--- a/docs/reference/connector/apis/list-connectors-api.asciidoc
+++ b/docs/reference/connector/apis/list-connectors-api.asciidoc
@@ -9,6 +9,8 @@ beta::[]
 
 Returns information about all created connectors.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 
 [[list-connector-api-request]]
 ==== {api-request-title}
diff --git a/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc
index 97b5c20f0813c..42203ed8e6103 100644
--- a/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc
+++ b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc
@@ -8,6 +8,8 @@ preview::[]
 
 Sets a connector sync job error.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[set-connector-sync-job-error-api-request]]
 ==== {api-request-title}
 `PUT _connector/_sync_job/<connector_sync_job_id>/_error`
diff --git a/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc b/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc
index 405df07465a28..4dd9cc6e67ab2 100644
--- a/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc
+++ b/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc
@@ -8,6 +8,8 @@ preview::[]
 
 Sets connector sync job stats.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[set-connector-sync-job-stats-api-request]]
 ==== {api-request-title}
 `PUT _connector/_sync_job/<connector_sync_job_id>/_stats`
diff --git a/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc b/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc
index b2b9dd7958191..112ec821df7c9 100644
--- a/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc
@@ -15,6 +15,8 @@ The Connector Secret ID is only required for native connectors.
 Connector clients do not use this field.
 See the documentation for {enterprise-search-ref}/native-connectors.html#native-connectors-manage-API-keys-programmatically[managing native connector API keys programmatically^] for more details.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[update-connector-api-key-id-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc
index 51252be4a04c3..e8a710cdacff0 100644
--- a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc
@@ -8,6 +8,7 @@ beta::[]
 
 Updates a connector's `configuration`, allowing for config value updates within a registered configuration schema.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
 
 [[update-connector-configuration-api-request]]
 ==== {api-request-title}
diff --git a/docs/reference/connector/apis/update-connector-error-api.asciidoc b/docs/reference/connector/apis/update-connector-error-api.asciidoc
index 3ec03c6153f4b..67ea6b6d17cf0 100644
--- a/docs/reference/connector/apis/update-connector-error-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-error-api.asciidoc
@@ -8,6 +8,8 @@ preview::[]
 
 Updates the `error` field of a connector.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[update-connector-error-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
index 7ae80276d3151..861e72481a59a 100644
--- a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc
@@ -10,6 +10,8 @@ Updates the draft `filtering` configuration of a connector and marks the draft v
 
 The filtering property is used to configure sync rules (both basic and advanced) for a connector. Learn more in the {enterprise-search-ref}/sync-rules.html[sync rules documentation].
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[update-connector-filtering-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/update-connector-index-name-api.asciidoc b/docs/reference/connector/apis/update-connector-index-name-api.asciidoc
index 3f931d8ac5edd..d07007438e09c 100644
--- a/docs/reference/connector/apis/update-connector-index-name-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-index-name-api.asciidoc
@@ -8,6 +8,8 @@ beta::[]
 
 Updates the `index_name` field of a connector, specifying the index where the data ingested by the connector is stored.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[update-connector-index-name-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc b/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc
index 9326855d3c5d8..918bf4f80a010 100644
--- a/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc
@@ -10,6 +10,8 @@ Updates the fields related to the last sync of a connector.
 
 This action is used for analytics and monitoring.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[update-connector-last-sync-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/update-connector-name-description-api.asciidoc b/docs/reference/connector/apis/update-connector-name-description-api.asciidoc
index 9ecf0f5d175fd..7e16874da9fb4 100644
--- a/docs/reference/connector/apis/update-connector-name-description-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-name-description-api.asciidoc
@@ -9,6 +9,8 @@ beta::[]
 
 Updates the `name` and `description` fields of a connector.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[update-connector-name-description-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc b/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc
index f8f804595eb19..01ed2e39702ea 100644
--- a/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc
@@ -10,6 +10,8 @@ Updates the `pipeline` configuration of a connector.
 
 When you create a new connector, the configuration of an <<ingest-pipeline-search-details-generic-reference, ingest pipeline>> is populated with default settings.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[update-connector-pipeline-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc b/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc
index 6d30197dd3390..f932f4c959de2 100644
--- a/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc
@@ -8,6 +8,8 @@ beta::[]
 
 Updates the `scheduling` configuration of a connector.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[update-connector-scheduling-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/update-connector-service-type-api.asciidoc b/docs/reference/connector/apis/update-connector-service-type-api.asciidoc
index fb61a25848a9d..139e9eddf4076 100644
--- a/docs/reference/connector/apis/update-connector-service-type-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-service-type-api.asciidoc
@@ -8,6 +8,8 @@ beta::[]
 
 Updates the `service_type` of a connector.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[update-connector-service-type-api-request]]
 ==== {api-request-title}
 
diff --git a/docs/reference/connector/apis/update-connector-status-api.asciidoc b/docs/reference/connector/apis/update-connector-status-api.asciidoc
index d444c82a1bde0..ee9dfcb5f880f 100644
--- a/docs/reference/connector/apis/update-connector-status-api.asciidoc
+++ b/docs/reference/connector/apis/update-connector-status-api.asciidoc
@@ -8,6 +8,8 @@ preview::[]
 
 Updates the `status` of a connector.
 
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
 [[update-connector-status-api-request]]
 ==== {api-request-title}
 

From 67a8bd219c9de211f904e5c9fcea4502d2637137 Mon Sep 17 00:00:00 2001
From: Artem Prigoda <arteamon@gmail.com>
Date: Wed, 29 May 2024 11:58:47 +0200
Subject: [PATCH 027/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109154

---
 muted-tests.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index 4f64492466375..9d78a6714e35e 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -35,6 +35,10 @@ tests:
 - class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108993"
   method: "testTrainedModelInference"
+- class: "org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109154"
+  method: "test {p0=data_stream/200_rollover_failure_store/Lazily roll over a data\
+    \ stream's failure store after an ingest failure}"
 # Examples:
 #
 #  Mute a single test case in a YAML test suite:

From f94c5416e30179a4b6cc0d2f564e96ea0961f800 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Wed, 29 May 2024 11:18:24 +0100
Subject: [PATCH 028/208] Fix trappy timeouts in CCR (#109151)

Relates #107984
---
 .../elasticsearch/xpack/ccr/AutoFollowIT.java | 35 +++++++++----
 .../elasticsearch/xpack/ccr/CcrLicenseIT.java |  5 +-
 .../xpack/ccr/CcrRetentionLeaseIT.java        | 19 +++++--
 .../xpack/ccr/CloseFollowerIndexIT.java       |  2 +-
 .../elasticsearch/xpack/ccr/FollowInfoIT.java | 32 +++++++-----
 .../xpack/ccr/FollowStatsIT.java              | 24 ++++++---
 .../xpack/ccr/IndexFollowingIT.java           | 51 +++++++++++++++----
 .../xpack/ccr/LocalIndexFollowingIT.java      | 10 ++--
 .../xpack/ccr/RestartIndexFollowingIT.java    |  6 ++-
 .../ccr/action/AutoFollowCoordinator.java     |  3 +-
 .../ccr/action/TransportPutFollowAction.java  |  3 +-
 .../xpack/ccr/rest/RestCcrStatsAction.java    |  3 +-
 .../RestDeleteAutoFollowPatternAction.java    |  4 +-
 .../xpack/ccr/rest/RestFollowInfoAction.java  |  3 +-
 .../rest/RestGetAutoFollowPatternAction.java  |  3 +-
 .../RestPauseAutoFollowPatternAction.java     |  4 +-
 .../xpack/ccr/rest/RestPauseFollowAction.java |  3 +-
 .../rest/RestPutAutoFollowPatternAction.java  |  5 +-
 .../xpack/ccr/rest/RestPutFollowAction.java   |  4 +-
 .../RestResumeAutoFollowPatternAction.java    |  4 +-
 .../ccr/rest/RestResumeFollowAction.java      |  8 ++-
 .../xpack/ccr/rest/RestUnfollowAction.java    |  8 ++-
 .../elasticsearch/xpack/CcrIntegTestCase.java | 12 +++--
 .../xpack/CcrSingleNodeTestCase.java          |  6 +--
 ...teAutoFollowPatternActionRequestTests.java | 16 ++++--
 .../action/AutoFollowCoordinatorTests.java    | 24 +++++++--
 .../DeleteAutoFollowPatternRequestTests.java  |  2 +-
 .../ccr/action/FollowInfoRequestTests.java    |  2 +-
 .../GetAutoFollowPatternRequestTests.java     |  2 +-
 .../PutAutoFollowPatternRequestTests.java     | 18 +++----
 .../action/PutFollowActionRequestTests.java   |  8 +--
 .../ResumeFollowActionRequestTests.java       |  8 +--
 ...tActivateAutoFollowPatternActionTests.java | 19 +++++--
 ...ortDeleteAutoFollowPatternActionTests.java |  6 +--
 ...nsportPutAutoFollowPatternActionTests.java |  6 +--
 .../TransportResumeFollowActionTests.java     |  2 +-
 .../ActivateAutoFollowPatternAction.java      |  5 +-
 .../xpack/core/ccr/action/CcrStatsAction.java |  4 +-
 .../action/DeleteAutoFollowPatternAction.java |  5 +-
 .../core/ccr/action/FollowInfoAction.java     |  5 +-
 .../action/GetAutoFollowPatternAction.java    |  5 +-
 .../core/ccr/action/PauseFollowAction.java    |  5 +-
 .../action/PutAutoFollowPatternAction.java    | 10 ++--
 .../core/ccr/action/PutFollowAction.java      | 10 ++--
 .../core/ccr/action/ResumeFollowAction.java   | 10 ++--
 .../xpack/core/ccr/action/UnfollowAction.java |  5 +-
 .../core/ilm/PauseFollowerIndexStep.java      |  3 +-
 .../core/ilm/UnfollowFollowerIndexStep.java   |  2 +-
 .../core/ccr/action/CcrStatsActionTests.java  |  8 +--
 .../collector/ccr/StatsCollector.java         |  2 +-
 50 files changed, 290 insertions(+), 159 deletions(-)

diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
index 8842d9ef35fec..bbd1905374d24 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
@@ -273,7 +273,7 @@ public void testAutoFollowParameterAreDelegated() throws Exception {
             .build();
 
         // Enabling auto following:
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setRemoteCluster("leader_cluster");
         request.setLeaderIndexPatterns(Collections.singletonList("logs-*"));
         // Need to set this, because following an index in the same cluster
@@ -314,7 +314,7 @@ public void testAutoFollowParameterAreDelegated() throws Exception {
 
         createLeaderIndex("logs-201901", leaderIndexSettings);
         assertLongBusy(() -> {
-            FollowInfoAction.Request followInfoRequest = new FollowInfoAction.Request();
+            FollowInfoAction.Request followInfoRequest = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT);
             followInfoRequest.setFollowerIndices("copy-logs-201901");
             FollowInfoAction.Response followInfoResponse;
             try {
@@ -662,7 +662,10 @@ public void testAutoFollowDatastreamWithClosingFollowerIndex() throws Exception
             .setSource("foo", "bar", DataStream.TIMESTAMP_FIELD_NAME, randomNonNegativeLong())
             .get();
 
-        PutAutoFollowPatternAction.Request followRequest = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request followRequest = new PutAutoFollowPatternAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT
+        );
         followRequest.setName("pattern-1");
         followRequest.setRemoteCluster("leader_cluster");
         followRequest.setLeaderIndexPatterns(List.of("logs-*"));
@@ -727,7 +730,7 @@ private void putAutoFollowPatterns(String name, String[] patterns) {
     }
 
     private void putAutoFollowPatterns(String name, String[] patterns, List<String> exclusionPatterns) {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setName(name);
         request.setRemoteCluster("leader_cluster");
         request.setLeaderIndexPatterns(Arrays.asList(patterns));
@@ -742,7 +745,11 @@ private void putAutoFollowPatterns(String name, String[] patterns, List<String>
     }
 
     private void deleteAutoFollowPattern(final String name) {
-        DeleteAutoFollowPatternAction.Request request = new DeleteAutoFollowPatternAction.Request(name);
+        DeleteAutoFollowPatternAction.Request request = new DeleteAutoFollowPatternAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            name
+        );
         if (randomBoolean()) {
             request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
@@ -750,7 +757,7 @@ private void deleteAutoFollowPattern(final String name) {
     }
 
     private AutoFollowStats getAutoFollowStats() {
-        CcrStatsAction.Request request = new CcrStatsAction.Request();
+        CcrStatsAction.Request request = new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT);
         if (randomBoolean()) {
             request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
@@ -764,7 +771,12 @@ private void createLeaderIndex(String index, Settings settings) {
     }
 
     private void pauseAutoFollowPattern(final String name) {
-        ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(name, false);
+        ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            name,
+            false
+        );
         if (randomBoolean()) {
             request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
@@ -772,7 +784,12 @@ private void pauseAutoFollowPattern(final String name) {
     }
 
     private void resumeAutoFollowPattern(final String name) {
-        ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(name, true);
+        ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            name,
+            true
+        );
         if (randomBoolean()) {
             request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
         }
@@ -780,7 +797,7 @@ private void resumeAutoFollowPattern(final String name) {
     }
 
     private AutoFollowMetadata.AutoFollowPattern getAutoFollowPattern(final String name) {
-        GetAutoFollowPatternAction.Request request = new GetAutoFollowPatternAction.Request();
+        GetAutoFollowPatternAction.Request request = new GetAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT);
         request.setName(name);
         if (randomBoolean()) {
             request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30)));
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java
index 549c5146e6c79..f1febd8aea550 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java
@@ -111,7 +111,10 @@ public void onFailure(final Exception e) {
 
     public void testThatPutAutoFollowPatternsIsUnavailableWithNonCompliantLicense() throws InterruptedException {
         final CountDownLatch latch = new CountDownLatch(1);
-        final PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        final PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT
+        );
         request.setName("name");
         request.setRemoteCluster("leader");
         request.setLeaderIndexPatterns(Collections.singletonList("*"));
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
index 339662c996492..a0b25faea9256 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
@@ -472,7 +472,12 @@ public void testUnfollowRemovesRetentionLeases() throws Exception {
 
             pauseFollow(followerIndex);
             assertAcked(followerClient().admin().indices().close(new CloseIndexRequest(followerIndex)).actionGet());
-            assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request(followerIndex)).actionGet());
+            assertAcked(
+                followerClient().execute(
+                    UnfollowAction.INSTANCE,
+                    new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, followerIndex)
+                ).actionGet()
+            );
 
             final IndicesStatsResponse afterUnfollowStats = leaderClient().admin()
                 .indices()
@@ -541,7 +546,10 @@ public void testUnfollowFailsToRemoveRetentionLeases() throws Exception {
 
             final ElasticsearchException e = expectThrows(
                 ElasticsearchException.class,
-                () -> followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request(followerIndex)).actionGet()
+                () -> followerClient().execute(
+                    UnfollowAction.INSTANCE,
+                    new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, followerIndex)
+                ).actionGet()
             );
 
             final ClusterStateResponse followerIndexClusterState = followerClient().admin()
@@ -993,7 +1001,12 @@ public void onResponseReceived(final long responseRequestId, final Transport.Res
 
             pauseFollow(followerIndex);
             assertAcked(followerClient().admin().indices().close(new CloseIndexRequest(followerIndex)).actionGet());
-            assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request(followerIndex)).actionGet());
+            assertAcked(
+                followerClient().execute(
+                    UnfollowAction.INSTANCE,
+                    new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, followerIndex)
+                ).actionGet()
+            );
 
             unfollowLatch.countDown();
 
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java
index a2d45e443e18f..9e84cdac34008 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java
@@ -74,7 +74,7 @@ public void testCloseAndReopenFollowerIndex() throws Exception {
         assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON));
         ensureLeaderYellow("index1");
 
-        PutFollowAction.Request followRequest = new PutFollowAction.Request();
+        PutFollowAction.Request followRequest = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         followRequest.setRemoteCluster("leader_cluster");
         followRequest.setLeaderIndex("index1");
         followRequest.setFollowerIndex("index2");
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowInfoIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowInfoIT.java
index efa7fcd0eec53..ab3b5f795d7a2 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowInfoIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowInfoIT.java
@@ -39,7 +39,7 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception {
         followRequest = getPutFollowRequest("leader2", "follower2");
         client().execute(PutFollowAction.INSTANCE, followRequest).get();
 
-        FollowInfoAction.Request request = new FollowInfoAction.Request();
+        FollowInfoAction.Request request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndices("follower1");
         FollowInfoAction.Response response = client().execute(FollowInfoAction.INSTANCE, request).actionGet();
         assertThat(response.getFollowInfos().size(), equalTo(1));
@@ -48,7 +48,7 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception {
         assertThat(response.getFollowInfos().get(0).getStatus(), equalTo(Status.ACTIVE));
         assertThat(response.getFollowInfos().get(0).getParameters(), notNullValue());
 
-        request = new FollowInfoAction.Request();
+        request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndices("follower2");
         response = client().execute(FollowInfoAction.INSTANCE, request).actionGet();
         assertThat(response.getFollowInfos().size(), equalTo(1));
@@ -57,7 +57,7 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception {
         assertThat(response.getFollowInfos().get(0).getStatus(), equalTo(Status.ACTIVE));
         assertThat(response.getFollowInfos().get(0).getParameters(), notNullValue());
 
-        request = new FollowInfoAction.Request();
+        request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndices("_all");
         response = client().execute(FollowInfoAction.INSTANCE, request).actionGet();
         response.getFollowInfos().sort(Comparator.comparing(FollowInfoAction.Response.FollowerInfo::getFollowerIndex));
@@ -72,9 +72,11 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception {
         assertThat(response.getFollowInfos().get(1).getParameters(), notNullValue());
 
         // Pause follower1 index and check the follower info api:
-        assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet());
+        assertAcked(
+            client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower1")).actionGet()
+        );
 
-        request = new FollowInfoAction.Request();
+        request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndices("follower1");
         response = client().execute(FollowInfoAction.INSTANCE, request).actionGet();
         assertThat(response.getFollowInfos().size(), equalTo(1));
@@ -83,7 +85,7 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception {
         assertThat(response.getFollowInfos().get(0).getStatus(), equalTo(Status.PAUSED));
         assertThat(response.getFollowInfos().get(0).getParameters(), nullValue());
 
-        request = new FollowInfoAction.Request();
+        request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndices("follower2");
         response = client().execute(FollowInfoAction.INSTANCE, request).actionGet();
         assertThat(response.getFollowInfos().size(), equalTo(1));
@@ -92,7 +94,7 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception {
         assertThat(response.getFollowInfos().get(0).getStatus(), equalTo(Status.ACTIVE));
         assertThat(response.getFollowInfos().get(0).getParameters(), notNullValue());
 
-        request = new FollowInfoAction.Request();
+        request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndices("_all");
         response = client().execute(FollowInfoAction.INSTANCE, request).actionGet();
         response.getFollowInfos().sort(Comparator.comparing(FollowInfoAction.Response.FollowerInfo::getFollowerIndex));
@@ -106,7 +108,9 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception {
         assertThat(response.getFollowInfos().get(1).getStatus(), equalTo(Status.ACTIVE));
         assertThat(response.getFollowInfos().get(1).getParameters(), notNullValue());
 
-        assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower2")).actionGet());
+        assertAcked(
+            client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower2")).actionGet()
+        );
     }
 
     public void testFollowInfoApiIndexMissing() throws Exception {
@@ -122,16 +126,20 @@ public void testFollowInfoApiIndexMissing() throws Exception {
         followRequest = getPutFollowRequest("leader2", "follower2");
         client().execute(PutFollowAction.INSTANCE, followRequest).get();
 
-        FollowInfoAction.Request request1 = new FollowInfoAction.Request();
+        FollowInfoAction.Request request1 = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT);
         request1.setFollowerIndices("follower3");
         expectThrows(IndexNotFoundException.class, () -> client().execute(FollowInfoAction.INSTANCE, request1).actionGet());
 
-        FollowInfoAction.Request request2 = new FollowInfoAction.Request();
+        FollowInfoAction.Request request2 = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT);
         request2.setFollowerIndices("follower2", "follower3");
         expectThrows(IndexNotFoundException.class, () -> client().execute(FollowInfoAction.INSTANCE, request2).actionGet());
 
-        assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet());
-        assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower2")).actionGet());
+        assertAcked(
+            client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower1")).actionGet()
+        );
+        assertAcked(
+            client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower2")).actionGet()
+        );
     }
 
 }
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java
index ccf1c72a06178..9fcee30d30b39 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java
@@ -100,14 +100,18 @@ public void testFollowStatsApiFollowerIndexFiltering() throws Exception {
         assertThat(response.getStatsResponses().get(0).status().followerIndex(), equalTo("follower1"));
         assertThat(response.getStatsResponses().get(1).status().followerIndex(), equalTo("follower2"));
 
-        assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet());
-        assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower2")).actionGet());
+        assertAcked(
+            client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower1")).actionGet()
+        );
+        assertAcked(
+            client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower2")).actionGet()
+        );
 
         assertBusy(() -> {
-            List<FollowStatsAction.StatsResponse> responseList = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request())
-                .actionGet()
-                .getFollowStats()
-                .getStatsResponses();
+            List<FollowStatsAction.StatsResponse> responseList = client().execute(
+                CcrStatsAction.INSTANCE,
+                new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT)
+            ).actionGet().getFollowStats().getStatsResponses();
             assertThat(responseList.size(), equalTo(0));
         });
     }
@@ -139,7 +143,9 @@ public void testFollowStatsApiResourceNotFound() throws Exception {
         e = expectThrows(ResourceNotFoundException.class, () -> client().execute(FollowStatsAction.INSTANCE, statsRequest).actionGet());
         assertThat(e.getMessage(), equalTo("No shard follow tasks for follower indices [follower2]"));
 
-        assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet());
+        assertAcked(
+            client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower1")).actionGet()
+        );
     }
 
     public void testFollowStatsApiWithDeletedFollowerIndex() throws Exception {
@@ -202,7 +208,9 @@ public void testFollowStatsApiIncludeShardFollowStatsWithClosedFollowerIndex() t
         assertThat(response.getStatsResponses().size(), equalTo(1));
         assertThat(response.getStatsResponses().get(0).status().followerIndex(), equalTo("follower1"));
 
-        assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet());
+        assertAcked(
+            client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower1")).actionGet()
+        );
     }
 
 }
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
index 097592a03d5d0..6361b6f89605e 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
@@ -435,7 +435,12 @@ public void testDoNotAllowPutMappingToFollower() throws Exception {
         assertThat(forbiddenException.status(), equalTo(RestStatus.FORBIDDEN));
         pauseFollow("index-2");
         followerClient().admin().indices().close(new CloseIndexRequest("index-2")).actionGet();
-        assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request("index-2")).actionGet());
+        assertAcked(
+            followerClient().execute(
+                UnfollowAction.INSTANCE,
+                new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index-2")
+            ).actionGet()
+        );
         followerClient().admin().indices().open(new OpenIndexRequest("index-2")).actionGet();
         assertAcked(followerClient().admin().indices().putMapping(putMappingRequest).actionGet());
     }
@@ -468,7 +473,12 @@ public void testAddAliasAfterUnfollow() throws Exception {
         followerClient().execute(PutFollowAction.INSTANCE, putFollow("leader", "follower")).get();
         pauseFollow("follower");
         followerClient().admin().indices().close(new CloseIndexRequest("follower").masterNodeTimeout(TimeValue.MAX_VALUE)).actionGet();
-        assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request("follower")).actionGet());
+        assertAcked(
+            followerClient().execute(
+                UnfollowAction.INSTANCE,
+                new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "follower")
+            ).actionGet()
+        );
         followerClient().admin().indices().open(new OpenIndexRequest("follower").masterNodeTimeout(TimeValue.MAX_VALUE)).actionGet();
         final IndicesAliasesRequest request = new IndicesAliasesRequest().masterNodeTimeout(TimeValue.MAX_VALUE)
             .addAliasAction(IndicesAliasesRequest.AliasActions.add().index("follower").alias("follower_alias"));
@@ -589,7 +599,7 @@ public void testFollowIndexWithNestedField() throws Exception {
     }
 
     public void testUnfollowNonExistingIndex() {
-        PauseFollowAction.Request unfollowRequest = new PauseFollowAction.Request("non-existing-index");
+        PauseFollowAction.Request unfollowRequest = new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "non-existing-index");
         expectThrows(IndexNotFoundException.class, () -> followerClient().execute(PauseFollowAction.INSTANCE, unfollowRequest).actionGet());
     }
 
@@ -899,25 +909,33 @@ public void testPauseIndex() throws Exception {
         );
         followerClient().execute(PutFollowAction.INSTANCE, putFollow("leader", "follower")).get();
         assertAcked(followerClient().admin().indices().prepareCreate("regular-index").setMasterNodeTimeout(TimeValue.MAX_VALUE));
-        assertAcked(followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower")).actionGet());
+        assertAcked(
+            followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower"))
+                .actionGet()
+        );
         assertThat(
             expectThrows(
                 IllegalArgumentException.class,
-                () -> followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower")).actionGet()
+                () -> followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower"))
+                    .actionGet()
             ).getMessage(),
             equalTo("no shard follow tasks for [follower]")
         );
         assertThat(
             expectThrows(
                 IllegalArgumentException.class,
-                () -> followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("regular-index")).actionGet()
+                () -> followerClient().execute(
+                    PauseFollowAction.INSTANCE,
+                    new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "regular-index")
+                ).actionGet()
             ).getMessage(),
             equalTo("index [regular-index] is not a follower index")
         );
         assertThat(
             expectThrows(
                 IndexNotFoundException.class,
-                () -> followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("xyz")).actionGet()
+                () -> followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "xyz"))
+                    .actionGet()
             ).getMessage(),
             equalTo("no such index [xyz]")
         );
@@ -937,7 +955,12 @@ public void testUnfollowIndex() throws Exception {
         // Turn follow index into a regular index by: pausing shard follow, close index, unfollow index and then open index:
         pauseFollow("index2");
         followerClient().admin().indices().close(new CloseIndexRequest("index2").masterNodeTimeout(TimeValue.MAX_VALUE)).actionGet();
-        assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request("index2")).actionGet());
+        assertAcked(
+            followerClient().execute(
+                UnfollowAction.INSTANCE,
+                new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index2")
+            ).actionGet()
+        );
         followerClient().admin().indices().open(new OpenIndexRequest("index2").masterNodeTimeout(TimeValue.MAX_VALUE)).actionGet();
         ensureFollowerGreen("index2");
 
@@ -960,7 +983,10 @@ public void testUnknownClusterAlias() throws Exception {
             () -> followerClient().execute(PutFollowAction.INSTANCE, followRequest).actionGet()
         );
         assertThat(e.getMessage(), equalTo("no such remote cluster: [another_cluster]"));
-        PutAutoFollowPatternAction.Request putAutoFollowRequest = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request putAutoFollowRequest = new PutAutoFollowPatternAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT
+        );
         putAutoFollowRequest.setName("name");
         putAutoFollowRequest.setRemoteCluster("another_cluster");
         putAutoFollowRequest.setLeaderIndexPatterns(Collections.singletonList("logs-*"));
@@ -1447,7 +1473,12 @@ private void runFallBehindTest(
         followerClient().admin().indices().prepareClose("index2").setMasterNodeTimeout(TimeValue.MAX_VALUE).get();
         pauseFollow("index2");
         if (randomBoolean()) {
-            assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request("index2")).actionGet());
+            assertAcked(
+                followerClient().execute(
+                    UnfollowAction.INSTANCE,
+                    new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index2")
+                ).actionGet()
+            );
         }
 
         final PutFollowAction.Request followRequest2 = putFollow("index1", "index2");
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java
index 05fc3b037c795..41fed34ea2106 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java
@@ -63,7 +63,7 @@ public void testFollowIndex() throws Exception {
 
         assertBusy(() -> assertHitCount(client().prepareSearch("follower"), firstBatchNumDocs + secondBatchNumDocs));
 
-        PauseFollowAction.Request pauseRequest = new PauseFollowAction.Request("follower");
+        PauseFollowAction.Request pauseRequest = new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower");
         client().execute(PauseFollowAction.INSTANCE, pauseRequest);
 
         final long thirdBatchNumDocs = randomIntBetween(2, 64);
@@ -136,7 +136,7 @@ public void testIndexingMetricsIncremented() throws Exception {
     }
 
     public void testRemoveRemoteConnection() throws Exception {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setName("my_pattern");
         request.setRemoteCluster("local");
         request.setLeaderIndexPatterns(Collections.singletonList("logs-*"));
@@ -152,7 +152,8 @@ public void testRemoveRemoteConnection() throws Exception {
         createIndex("logs-20200101", leaderIndexSettings);
         prepareIndex("logs-20200101").setSource("{}", XContentType.JSON).get();
         assertBusy(() -> {
-            CcrStatsAction.Response response = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()).actionGet();
+            CcrStatsAction.Response response = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT))
+                .actionGet();
             assertThat(
                 response.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(),
                 equalTo(previousNumberOfSuccessfulFollowedIndices + 1)
@@ -171,7 +172,8 @@ public void testRemoveRemoteConnection() throws Exception {
         // This new document should be replicated to follower index:
         prepareIndex("logs-20200101").setSource("{}", XContentType.JSON).get();
         assertBusy(() -> {
-            CcrStatsAction.Response response = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()).actionGet();
+            CcrStatsAction.Response response = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT))
+                .actionGet();
             assertThat(
                 response.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(),
                 equalTo(previousNumberOfSuccessfulFollowedIndices + 2)
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java
index 5c152be35b509..18456b24d4618 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java
@@ -118,12 +118,14 @@ public void testFollowIndex() throws Exception {
         }, 30L, TimeUnit.SECONDS);
 
         cleanRemoteCluster();
-        assertAcked(followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("index2")).actionGet());
+        assertAcked(
+            followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "index2")).actionGet()
+        );
         assertAcked(followerClient().admin().indices().prepareClose("index2"));
 
         final ActionFuture<AcknowledgedResponse> unfollowFuture = followerClient().execute(
             UnfollowAction.INSTANCE,
-            new UnfollowAction.Request("index2")
+            new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index2")
         );
         final ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, unfollowFuture::actionGet);
         assertThat(elasticsearchException.getMessage(), containsString("no such remote cluster"));
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
index a7bf572e9bf73..82af24d2293cc 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
@@ -712,7 +712,8 @@ static PutFollowAction.Request generateRequest(
             final String leaderIndexName = indexToFollow.getName();
             final String followIndexName = getFollowerIndexName(pattern, leaderIndexName);
 
-            PutFollowAction.Request request = new PutFollowAction.Request();
+            // TODO use longer timeouts here? see https://github.com/elastic/elasticsearch/issues/109150
+            PutFollowAction.Request request = new PutFollowAction.Request(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS);
             request.setRemoteCluster(remoteCluster);
             request.setLeaderIndex(indexToFollow.getName());
             request.setFollowerIndex(followIndexName);
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
index a0917c1cef815..ed7587556bd28 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
@@ -294,10 +294,9 @@ private void initiateFollowing(
     ) {
         assert request.waitForActiveShards() != ActiveShardCount.DEFAULT : "PutFollowAction does not support DEFAULT.";
         FollowParameters parameters = request.getParameters();
-        ResumeFollowAction.Request resumeFollowRequest = new ResumeFollowAction.Request();
+        ResumeFollowAction.Request resumeFollowRequest = new ResumeFollowAction.Request(request.masterNodeTimeout());
         resumeFollowRequest.setFollowerIndex(request.getFollowerIndex());
         resumeFollowRequest.setParameters(new FollowParameters(parameters));
-        resumeFollowRequest.masterNodeTimeout(request.masterNodeTimeout());
         clientWithHeaders.execute(
             ResumeFollowAction.INSTANCE,
             resumeFollowRequest,
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java
index 7c8ebc5a66e80..ff208b0b92f88 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java
@@ -38,11 +38,10 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
-        final CcrStatsAction.Request request = new CcrStatsAction.Request();
+        final CcrStatsAction.Request request = new CcrStatsAction.Request(getMasterNodeTimeout(restRequest));
         if (restRequest.hasParam("timeout")) {
             request.setTimeout(restRequest.paramAsTime("timeout", null));
         }
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
         return channel -> client.execute(
             CcrStatsAction.INSTANCE,
             request,
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java
index 1f96ea6be9dc5..65c00d94635c5 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.ccr.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -32,8 +33,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        Request request = new Request(restRequest.param("name"));
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        final var request = new Request(getMasterNodeTimeout(restRequest), TimeValue.THIRTY_SECONDS, restRequest.param("name"));
         return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel));
     }
 
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java
index e20c34fe38243..12fb8690adf66 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java
@@ -33,8 +33,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
-        final FollowInfoAction.Request request = new FollowInfoAction.Request();
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        final var request = new FollowInfoAction.Request(getMasterNodeTimeout(restRequest));
         request.setFollowerIndices(Strings.splitStringByCommaToArray(restRequest.param("index")));
         return channel -> client.execute(FollowInfoAction.INSTANCE, request, new RestRefCountedChunkedToXContentListener<>(channel));
     }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestGetAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestGetAutoFollowPatternAction.java
index 84a8d4f879e02..bb202d74d185a 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestGetAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestGetAutoFollowPatternAction.java
@@ -32,9 +32,8 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        Request request = new Request();
+        final var request = new Request(getMasterNodeTimeout(restRequest));
         request.setName(restRequest.param("name"));
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
         return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel));
     }
 
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseAutoFollowPatternAction.java
index 5a2ba2fe736f7..b27960d5bc4ab 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseAutoFollowPatternAction.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.ccr.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -32,8 +33,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
-        Request request = new Request(restRequest.param("name"), false);
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        final var request = new Request(getMasterNodeTimeout(restRequest), TimeValue.THIRTY_SECONDS, restRequest.param("name"), false);
         return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseFollowAction.java
index 8c0f79f0b2440..a5e967ba7ac5a 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseFollowAction.java
@@ -32,8 +32,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        Request request = new Request(restRequest.param("index"));
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        final var request = new Request(getMasterNodeTimeout(restRequest), restRequest.param("index"));
         return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutAutoFollowPatternAction.java
index cb42431022501..dedcb941483b0 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutAutoFollowPatternAction.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.ccr.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -40,9 +41,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
 
     private static Request createRequest(RestRequest restRequest) throws IOException {
         try (XContentParser parser = restRequest.contentOrSourceParamParser()) {
-            Request request = Request.fromXContent(parser, restRequest.param("name"));
-            request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
-            return request;
+            return Request.fromXContent(getMasterNodeTimeout(restRequest), TimeValue.THIRTY_SECONDS, parser, restRequest.param("name"));
         }
     }
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java
index 162431d68fb0f..d2063657c8784 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -41,10 +42,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
 
     private static Request createRequest(RestRequest restRequest) throws IOException {
         try (XContentParser parser = restRequest.contentOrSourceParamParser()) {
-            final Request request = Request.fromXContent(parser);
+            final var request = Request.fromXContent(getMasterNodeTimeout(restRequest), TimeValue.THIRTY_SECONDS, parser);
             request.waitForActiveShards(ActiveShardCount.parseString(restRequest.param("wait_for_active_shards")));
             request.setFollowerIndex(restRequest.param("index"));
-            request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
             return request;
         }
     }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeAutoFollowPatternAction.java
index 3e51386ef1069..13ef9a6d751f7 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeAutoFollowPatternAction.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.ccr.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -32,8 +33,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
-        Request request = new Request(restRequest.param("name"), true);
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        final var request = new Request(getMasterNodeTimeout(restRequest), TimeValue.THIRTY_SECONDS, restRequest.param("name"), true);
         return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeFollowAction.java
index 86a00ca1ff020..b519d76c841d2 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeFollowAction.java
@@ -39,16 +39,14 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
     }
 
     static Request createRequest(RestRequest restRequest) throws IOException {
-        Request request;
         if (restRequest.hasContentOrSourceParam()) {
             try (XContentParser parser = restRequest.contentOrSourceParamParser()) {
-                request = Request.fromXContent(parser, restRequest.param("index"));
+                return Request.fromXContent(getMasterNodeTimeout(restRequest), parser, restRequest.param("index"));
             }
         } else {
-            request = new Request();
+            final var request = new Request(getMasterNodeTimeout(restRequest));
             request.setFollowerIndex(restRequest.param("index"));
+            return request;
         }
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
-        return request;
     }
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestUnfollowAction.java
index acc6ffb0a67bd..57e69b686b268 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestUnfollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestUnfollowAction.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.ccr.rest;
 
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -33,8 +34,11 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
-        UnfollowAction.Request request = new UnfollowAction.Request(restRequest.param("index"));
-        request.masterNodeTimeout(getMasterNodeTimeout(restRequest));
+        UnfollowAction.Request request = new UnfollowAction.Request(
+            getMasterNodeTimeout(restRequest),
+            TimeValue.THIRTY_SECONDS,
+            restRequest.param("index")
+        );
         return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel));
     }
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
index 52343be3f2c23..677a82ddafa34 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
@@ -484,7 +484,7 @@ protected void ensureEmptyWriteBuffers() throws Exception {
 
     protected void pauseFollow(String... indices) throws Exception {
         for (String index : indices) {
-            final PauseFollowAction.Request unfollowRequest = new PauseFollowAction.Request(index);
+            final PauseFollowAction.Request unfollowRequest = new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, index);
             assertAcked(followerClient().execute(PauseFollowAction.INSTANCE, unfollowRequest).actionGet());
         }
         ensureNoCcrTasks();
@@ -492,8 +492,10 @@ protected void pauseFollow(String... indices) throws Exception {
 
     protected void ensureNoCcrTasks() throws Exception {
         assertBusy(() -> {
-            CcrStatsAction.Response statsResponse = followerClient().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request())
-                .actionGet();
+            CcrStatsAction.Response statsResponse = followerClient().execute(
+                CcrStatsAction.INSTANCE,
+                new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT)
+            ).actionGet();
             assertThat(
                 "Follow stats not empty: " + Strings.toString(statsResponse.getFollowStats()),
                 statsResponse.getFollowStats().getStatsResponses(),
@@ -586,7 +588,7 @@ public static PutFollowAction.Request putFollow(String leaderIndex, String follo
     }
 
     public static PutFollowAction.Request putFollow(String leaderIndex, String followerIndex, ActiveShardCount waitForActiveShards) {
-        PutFollowAction.Request request = new PutFollowAction.Request();
+        PutFollowAction.Request request = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setRemoteCluster("leader_cluster");
         request.setLeaderIndex(leaderIndex);
         request.setFollowerIndex(followerIndex);
@@ -602,7 +604,7 @@ public static PutFollowAction.Request putFollow(String leaderIndex, String follo
     }
 
     public static ResumeFollowAction.Request resumeFollow(String followerIndex) {
-        ResumeFollowAction.Request request = new ResumeFollowAction.Request();
+        ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndex(followerIndex);
         request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(10));
         request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(10));
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
index efcaee96c008a..6b69c172c0df3 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
@@ -95,11 +95,11 @@ public void removeLocalRemote() throws Exception {
     }
 
     protected AutoFollowStats getAutoFollowStats() {
-        return client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()).actionGet().getAutoFollowStats();
+        return client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT)).actionGet().getAutoFollowStats();
     }
 
     protected ResumeFollowAction.Request getResumeFollowRequest(String followerIndex) {
-        ResumeFollowAction.Request request = new ResumeFollowAction.Request();
+        ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndex(followerIndex);
         request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(1));
         request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(1));
@@ -110,7 +110,7 @@ protected ResumeFollowAction.Request getResumeFollowRequest(String followerIndex
     }
 
     protected PutFollowAction.Request getPutFollowRequest(String leaderIndex, String followerIndex) {
-        PutFollowAction.Request request = new PutFollowAction.Request();
+        PutFollowAction.Request request = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setRemoteCluster("local");
         request.setLeaderIndex(leaderIndex);
         request.setFollowerIndex(followerIndex);
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java
index 03b197b629d02..f9386cf126a24 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java
@@ -19,7 +19,12 @@ public class ActivateAutoFollowPatternActionRequestTests extends AbstractWireSer
 
     @Override
     protected ActivateAutoFollowPatternAction.Request createTestInstance() {
-        return new ActivateAutoFollowPatternAction.Request(randomAlphaOfLength(5), randomBoolean());
+        return new ActivateAutoFollowPatternAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            randomAlphaOfLength(5),
+            randomBoolean()
+        );
     }
 
     @Override
@@ -33,12 +38,17 @@ protected Writeable.Reader<ActivateAutoFollowPatternAction.Request> instanceRead
     }
 
     public void testValidate() {
-        ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(null, true);
+        ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(
+            TEST_REQUEST_TIMEOUT,
+            TEST_REQUEST_TIMEOUT,
+            null,
+            true
+        );
         ActionRequestValidationException validationException = request.validate();
         assertThat(validationException, notNullValue());
         assertThat(validationException.getMessage(), containsString("[name] is missing"));
 
-        request = new ActivateAutoFollowPatternAction.Request("name", true);
+        request = new ActivateAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "name", true);
         validationException = request.validate();
         assertThat(validationException, nullValue());
     }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java
index f15ca19dd590a..467ef3c68f648 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java
@@ -460,7 +460,10 @@ public void testAutoFollowerWithPausedActivePatterns() {
             final ClusterState nextLocalClusterState;
             if (nextClusterStateVersion == 1) {
                 // cluster state #1 : one pattern is active
-                PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+                PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    TEST_REQUEST_TIMEOUT
+                );
                 request.setName("patternLogs");
                 request.setRemoteCluster(remoteCluster);
                 request.setLeaderIndexPatterns(singletonList("patternLogs-*"));
@@ -478,7 +481,10 @@ public void testAutoFollowerWithPausedActivePatterns() {
 
             } else if (nextClusterStateVersion == 3) {
                 // cluster state #3 : add a new pattern, two patterns are active
-                PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+                PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    TEST_REQUEST_TIMEOUT
+                );
                 request.setName("patternDocs");
                 request.setRemoteCluster(remoteCluster);
                 request.setLeaderIndexPatterns(singletonList("patternDocs-*"));
@@ -496,12 +502,22 @@ public void testAutoFollowerWithPausedActivePatterns() {
 
             } else if (nextClusterStateVersion == 5) {
                 // cluster state #5 : first pattern is paused, second pattern is still active
-                ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request("patternLogs", false);
+                ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    TEST_REQUEST_TIMEOUT,
+                    "patternLogs",
+                    false
+                );
                 nextLocalClusterState = TransportActivateAutoFollowPatternAction.innerActivate(request, currentLocalState);
 
             } else if (nextClusterStateVersion == 6) {
                 // cluster state #5 : second pattern is paused, both patterns are inactive
-                ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request("patternDocs", false);
+                ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(
+                    TEST_REQUEST_TIMEOUT,
+                    TEST_REQUEST_TIMEOUT,
+                    "patternDocs",
+                    false
+                );
                 nextLocalClusterState = TransportActivateAutoFollowPatternAction.innerActivate(request, currentLocalState);
 
             } else {
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java
index b4a28fd149326..1ca79401ae896 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java
@@ -19,7 +19,7 @@ protected Writeable.Reader<DeleteAutoFollowPatternAction.Request> instanceReader
 
     @Override
     protected DeleteAutoFollowPatternAction.Request createTestInstance() {
-        return new DeleteAutoFollowPatternAction.Request(randomAlphaOfLength(4));
+        return new DeleteAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomAlphaOfLength(4));
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java
index f08ab5aca1aa8..c809572195f94 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java
@@ -19,7 +19,7 @@ protected Writeable.Reader<FollowInfoAction.Request> instanceReader() {
 
     @Override
     protected FollowInfoAction.Request createTestInstance() {
-        FollowInfoAction.Request request = new FollowInfoAction.Request();
+        FollowInfoAction.Request request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndices(generateRandomStringArray(4, 4, true, false));
         return request;
     }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java
index b451e07bd037d..d6b498e0bd3f9 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java
@@ -19,7 +19,7 @@ protected Writeable.Reader<GetAutoFollowPatternAction.Request> instanceReader()
 
     @Override
     protected GetAutoFollowPatternAction.Request createTestInstance() {
-        GetAutoFollowPatternAction.Request request = new GetAutoFollowPatternAction.Request();
+        GetAutoFollowPatternAction.Request request = new GetAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT);
         if (randomBoolean()) {
             request.setName(randomAlphaOfLength(4));
         }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
index 9e2ca3d8b3013..93d363251da7e 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
@@ -26,7 +26,7 @@ public class PutAutoFollowPatternRequestTests extends AbstractXContentSerializin
 
     @Override
     protected PutAutoFollowPatternAction.Request doParseInstance(XContentParser parser) throws IOException {
-        return PutAutoFollowPatternAction.Request.fromXContent(parser, null);
+        return PutAutoFollowPatternAction.Request.fromXContent(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, parser, null);
     }
 
     @Override
@@ -36,7 +36,7 @@ protected Writeable.Reader<PutAutoFollowPatternAction.Request> instanceReader()
 
     @Override
     protected PutAutoFollowPatternAction.Request createTestInstance() {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setName(randomAlphaOfLength(4));
 
         request.setRemoteCluster(randomAlphaOfLength(4));
@@ -62,7 +62,7 @@ protected PutAutoFollowPatternAction.Request mutateInstance(PutAutoFollowPattern
     protected PutAutoFollowPatternAction.Request createXContextTestInstance(XContentType xContentType) {
         // follower index parameter is not part of the request body and is provided in the url path.
         // So this field cannot be used for creating a test instance for xcontent testing.
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setRemoteCluster(randomAlphaOfLength(4));
         request.setLeaderIndexPatterns(Arrays.asList(generateRandomStringArray(4, 4, false)));
         if (randomBoolean()) {
@@ -78,7 +78,7 @@ protected PutAutoFollowPatternAction.Request createXContextTestInstance(XContent
     }
 
     public void testValidate() {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         ActionRequestValidationException validationException = request.validate();
         assertThat(validationException, notNullValue());
         assertThat(validationException.getMessage(), containsString("[name] is missing"));
@@ -118,7 +118,7 @@ public void testValidate() {
     }
 
     public void testValidateName() {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setRemoteCluster("_alias");
         request.setLeaderIndexPatterns(Collections.singletonList("logs-*"));
 
@@ -128,7 +128,7 @@ public void testValidateName() {
     }
 
     public void testValidateNameComma() {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setRemoteCluster("_alias");
         request.setLeaderIndexPatterns(Collections.singletonList("logs-*"));
 
@@ -139,7 +139,7 @@ public void testValidateNameComma() {
     }
 
     public void testValidateNameLeadingUnderscore() {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setRemoteCluster("_alias");
         request.setLeaderIndexPatterns(Collections.singletonList("logs-*"));
 
@@ -150,7 +150,7 @@ public void testValidateNameLeadingUnderscore() {
     }
 
     public void testValidateNameUnderscores() {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setRemoteCluster("_alias");
         request.setLeaderIndexPatterns(Collections.singletonList("logs-*"));
 
@@ -160,7 +160,7 @@ public void testValidateNameUnderscores() {
     }
 
     public void testValidateNameTooLong() {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setRemoteCluster("_alias");
         request.setLeaderIndexPatterns(Collections.singletonList("logs-*"));
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java
index 171727c3e0bc8..4fa3c0a11acb0 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java
@@ -27,7 +27,7 @@ protected Writeable.Reader<PutFollowAction.Request> instanceReader() {
 
     @Override
     protected PutFollowAction.Request createTestInstance() {
-        PutFollowAction.Request request = new PutFollowAction.Request();
+        PutFollowAction.Request request = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setFollowerIndex(randomAlphaOfLength(4));
         request.waitForActiveShards(
             randomFrom(ActiveShardCount.DEFAULT, ActiveShardCount.NONE, ActiveShardCount.ONE, ActiveShardCount.ALL)
@@ -47,7 +47,7 @@ protected PutFollowAction.Request createTestInstance() {
     protected PutFollowAction.Request createXContextTestInstance(XContentType xContentType) {
         // follower index parameter and wait for active shards params are not part of the request body and
         // are provided in the url path. So these fields cannot be used for creating a test instance for xcontent testing.
-        PutFollowAction.Request request = new PutFollowAction.Request();
+        PutFollowAction.Request request = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setRemoteCluster(randomAlphaOfLength(4));
         request.setLeaderIndex(randomAlphaOfLength(4));
         request.setSettings(
@@ -61,7 +61,7 @@ protected PutFollowAction.Request createXContextTestInstance(XContentType xConte
 
     @Override
     protected PutFollowAction.Request doParseInstance(XContentParser parser) throws IOException {
-        PutFollowAction.Request request = PutFollowAction.Request.fromXContent(parser);
+        PutFollowAction.Request request = PutFollowAction.Request.fromXContent(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, parser);
         request.waitForActiveShards(ActiveShardCount.DEFAULT);
         request.setFollowerIndex("followerIndex");
         return request;
@@ -69,7 +69,7 @@ protected PutFollowAction.Request doParseInstance(XContentParser parser) throws
 
     @Override
     protected PutFollowAction.Request mutateInstance(PutFollowAction.Request instance) {
-        PutFollowAction.Request request = new PutFollowAction.Request();
+        PutFollowAction.Request request = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setFollowerIndex(instance.getFollowerIndex());
         request.waitForActiveShards(instance.waitForActiveShards());
         request.setRemoteCluster(instance.getRemoteCluster());
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java
index 8a835a4d1f715..475c9135d4946 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java
@@ -32,7 +32,7 @@ protected Writeable.Reader<ResumeFollowAction.Request> instanceReader() {
 
     @Override
     protected ResumeFollowAction.Request createTestInstance() {
-        ResumeFollowAction.Request request = new ResumeFollowAction.Request();
+        ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndex(randomAlphaOfLength(4));
 
         generateFollowParameters(request.getParameters());
@@ -48,14 +48,14 @@ protected ResumeFollowAction.Request mutateInstance(ResumeFollowAction.Request i
     protected ResumeFollowAction.Request createXContextTestInstance(XContentType type) {
         // follower index parameter is not part of the request body and is provided in the url path.
         // So this field cannot be used for creating a test instance for xcontent testing.
-        ResumeFollowAction.Request request = new ResumeFollowAction.Request();
+        ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT);
         generateFollowParameters(request.getParameters());
         return request;
     }
 
     @Override
     protected ResumeFollowAction.Request doParseInstance(XContentParser parser) throws IOException {
-        return ResumeFollowAction.Request.fromXContent(parser, null);
+        return ResumeFollowAction.Request.fromXContent(TEST_REQUEST_TIMEOUT, parser, null);
     }
 
     static void generateFollowParameters(FollowParameters followParameters) {
@@ -92,7 +92,7 @@ static void generateFollowParameters(FollowParameters followParameters) {
     }
 
     public void testValidate() {
-        ResumeFollowAction.Request request = new ResumeFollowAction.Request();
+        ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndex("index2");
         request.getParameters().setMaxRetryDelay(TimeValue.ZERO);
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java
index 206c71e82c52c..3a6a0d90f60ba 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java
@@ -33,7 +33,10 @@ public class TransportActivateAutoFollowPatternActionTests extends ESTestCase {
     public void testInnerActivateNoAutoFollowMetadata() {
         Exception e = expectThrows(
             ResourceNotFoundException.class,
-            () -> TransportActivateAutoFollowPatternAction.innerActivate(new Request("test", true), ClusterState.EMPTY_STATE)
+            () -> TransportActivateAutoFollowPatternAction.innerActivate(
+                new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "test", true),
+                ClusterState.EMPTY_STATE
+            )
         );
         assertThat(e.getMessage(), equalTo("auto-follow pattern [test] is missing"));
     }
@@ -54,7 +57,10 @@ public void testInnerActivateDoesNotExist() {
             .build();
         Exception e = expectThrows(
             ResourceNotFoundException.class,
-            () -> TransportActivateAutoFollowPatternAction.innerActivate(new Request("does_not_exist", true), clusterState)
+            () -> TransportActivateAutoFollowPatternAction.innerActivate(
+                new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "does_not_exist", true),
+                clusterState
+            )
         );
         assertThat(e.getMessage(), equalTo("auto-follow pattern [does_not_exist] is missing"));
     }
@@ -75,12 +81,17 @@ public void testInnerActivateToggle() {
             )
             .build();
         {
-            Request pauseRequest = new Request("remote_cluster", autoFollowPattern.isActive());
+            Request pauseRequest = new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "remote_cluster", autoFollowPattern.isActive());
             ClusterState updatedState = TransportActivateAutoFollowPatternAction.innerActivate(pauseRequest, clusterState);
             assertThat(updatedState, sameInstance(clusterState));
         }
         {
-            Request pauseRequest = new Request("remote_cluster", autoFollowPattern.isActive() == false);
+            Request pauseRequest = new Request(
+                TEST_REQUEST_TIMEOUT,
+                TEST_REQUEST_TIMEOUT,
+                "remote_cluster",
+                autoFollowPattern.isActive() == false
+            );
             ClusterState updatedState = TransportActivateAutoFollowPatternAction.innerActivate(pauseRequest, clusterState);
             assertThat(updatedState, not(sameInstance(clusterState)));
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java
index 470c15d86dfb1..f94f23c3695bf 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java
@@ -101,7 +101,7 @@ public void testInnerDelete() {
             )
             .build();
 
-        Request request = new Request("name1");
+        Request request = new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "name1");
         AutoFollowMetadata result = TransportDeleteAutoFollowPatternAction.innerDelete(request, clusterState)
             .getMetadata()
             .custom(AutoFollowMetadata.TYPE);
@@ -154,7 +154,7 @@ public void testInnerDeleteDoesNotExist() {
             )
             .build();
 
-        Request request = new Request("name2");
+        Request request = new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "name2");
         Exception e = expectThrows(
             ResourceNotFoundException.class,
             () -> TransportDeleteAutoFollowPatternAction.innerDelete(request, clusterState)
@@ -165,7 +165,7 @@ public void testInnerDeleteDoesNotExist() {
     public void testInnerDeleteNoAutoFollowMetadata() {
         ClusterState clusterState = ClusterState.builder(new ClusterName("us_cluster")).metadata(Metadata.builder()).build();
 
-        Request request = new Request("name1");
+        Request request = new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "name1");
         Exception e = expectThrows(
             ResourceNotFoundException.class,
             () -> TransportDeleteAutoFollowPatternAction.innerDelete(request, clusterState)
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java
index cd2bca527bfb9..ea37cdf1aae0c 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java
@@ -31,7 +31,7 @@
 public class TransportPutAutoFollowPatternActionTests extends ESTestCase {
 
     public void testInnerPut() {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setName("name1");
         request.setRemoteCluster("eu_cluster");
         request.setLeaderIndexPatterns(Collections.singletonList("logs-*"));
@@ -73,7 +73,7 @@ public void testInnerPut() {
     }
 
     public void testInnerPut_existingLeaderIndices() {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setName("name1");
         request.setRemoteCluster("eu_cluster");
         request.setLeaderIndexPatterns(Collections.singletonList("logs-*"));
@@ -129,7 +129,7 @@ public void testInnerPut_existingLeaderIndices() {
     }
 
     public void testInnerPut_existingLeaderIndicesAndAutoFollowMetadata() {
-        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request();
+        PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT);
         request.setName("name1");
         request.setRemoteCluster("eu_cluster");
         request.setLeaderIndexPatterns(Arrays.asList("logs-*", "transactions-*"));
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
index 1313e5781f122..7de0d775ba150 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
@@ -36,7 +36,7 @@
 public class TransportResumeFollowActionTests extends ESTestCase {
 
     public static ResumeFollowAction.Request resumeFollow(String followerIndex) {
-        ResumeFollowAction.Request request = new ResumeFollowAction.Request();
+        ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT);
         request.setFollowerIndex(followerIndex);
         request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(10));
         request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(10));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ActivateAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ActivateAutoFollowPatternAction.java
index df917b4e97b7d..acab22f620fa0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ActivateAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ActivateAutoFollowPatternAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -33,8 +34,8 @@ public static class Request extends AcknowledgedRequest<Request> {
         private final String name;
         private final boolean active;
 
-        public Request(final String name, final boolean active) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, final String name, final boolean active) {
+            super(masterNodeTimeout, ackTimeout);
             this.name = name;
             this.active = active;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
index b187e5e39dd33..a0ecc5dd566c2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
@@ -45,8 +45,8 @@ public Request(StreamInput in) throws IOException {
             }
         }
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout) {
+            super(masterNodeTimeout);
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java
index e38a1cfd4a2cb..cdacb184744b2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -31,8 +32,8 @@ public static class Request extends AcknowledgedRequest<Request> {
 
         private final String name;
 
-        public Request(String name) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String name) {
+            super(masterNodeTimeout, ackTimeout);
             this.name = name;
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java
index d979a4cf44b9b..821559e16fa38 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -41,8 +42,8 @@ public static class Request extends MasterNodeReadRequest<Request> {
 
         private String[] followerIndices;
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout) {
+            super(masterNodeTimeout);
         }
 
         public String[] getFollowerIndices() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java
index bd6ab5bb5af44..5f08a4c253005 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern;
@@ -34,8 +35,8 @@ public static class Request extends MasterNodeReadRequest<Request> {
 
         private String name;
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout) {
+            super(masterNodeTimeout);
         }
 
         public Request(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java
index c6905b2d06a34..54687001fbbbb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -30,8 +31,8 @@ public static class Request extends MasterNodeRequest<Request> {
 
         private final String followIndex;
 
-        public Request(String followIndex) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, String followIndex) {
+            super(masterNodeTimeout);
             this.followIndex = Objects.requireNonNull(followIndex, "followIndex");
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
index 333171d864c4f..dcee7274632eb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -62,9 +63,10 @@ public static class Request extends AcknowledgedRequest<Request> implements ToXC
             FollowParameters.initParser(PARSER);
         }
 
-        public static Request fromXContent(XContentParser parser, String name) throws IOException {
+        public static Request fromXContent(TimeValue masterNodeTimeout, TimeValue ackTimeout, XContentParser parser, String name)
+            throws IOException {
             PutAutoFollowPatternParameters parameters = PARSER.parse(parser, null);
-            Request request = new Request();
+            Request request = new Request(masterNodeTimeout, ackTimeout);
             request.setName(name);
             request.setRemoteCluster(parameters.remoteCluster);
             request.setLeaderIndexPatterns(parameters.leaderIndexPatterns);
@@ -85,8 +87,8 @@ public static Request fromXContent(XContentParser parser, String name) throws IO
         private FollowParameters parameters = new FollowParameters();
         private List<String> leaderIndexExclusionPatterns = Collections.emptyList();
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+            super(masterNodeTimeout, ackTimeout);
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
index db1e84aca9cda..82941c440484d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -64,10 +65,11 @@ public static final class Request extends AcknowledgedRequest<Request> implement
             FollowParameters.initParser(PARSER);
         }
 
-        public static Request fromXContent(final XContentParser parser) throws IOException {
+        public static Request fromXContent(TimeValue masterNodeTimeout, TimeValue ackTimeout, final XContentParser parser)
+            throws IOException {
             PutFollowParameters parameters = PARSER.parse(parser, null);
 
-            Request request = new Request();
+            Request request = new Request(masterNodeTimeout, ackTimeout);
             request.setRemoteCluster(parameters.remoteCluster);
             request.setLeaderIndex(parameters.leaderIndex);
             request.setDataStreamName(parameters.dataStreamName);
@@ -85,8 +87,8 @@ public static Request fromXContent(final XContentParser parser) throws IOExcepti
         private FollowParameters parameters = new FollowParameters();
         private ActiveShardCount waitForActiveShards = ActiveShardCount.NONE;
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout) {
+            super(masterNodeTimeout, ackTimeout);
         }
 
         public String getFollowerIndex() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java
index 12ddea8d99578..04c77dbbad952 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -43,9 +44,10 @@ public static class Request extends MasterNodeRequest<Request> implements ToXCon
             FollowParameters.initParser(PARSER);
         }
 
-        public static Request fromXContent(final XContentParser parser, final String followerIndex) throws IOException {
+        public static Request fromXContent(TimeValue masterNodeTimeout, final XContentParser parser, final String followerIndex)
+            throws IOException {
             FollowParameters parameters = PARSER.parse(parser, null);
-            Request request = new Request();
+            Request request = new Request(masterNodeTimeout);
             request.setFollowerIndex(followerIndex);
             request.setParameters(parameters);
             return request;
@@ -54,8 +56,8 @@ public static Request fromXContent(final XContentParser parser, final String fol
         private String followerIndex;
         private FollowParameters parameters = new FollowParameters();
 
-        public Request() {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout) {
+            super(masterNodeTimeout);
         }
 
         public String getFollowerIndex() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java
index 9a5f011f39a1b..1647b67a0e6ec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -33,8 +34,8 @@ public static class Request extends AcknowledgedRequest<Request> implements Indi
 
         private final String followerIndex;
 
-        public Request(String followerIndex) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String followerIndex) {
+            super(masterNodeTimeout, ackTimeout);
             this.followerIndex = followerIndex;
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java
index 5cecd0eb924f5..d900f168be06c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java
@@ -52,8 +52,7 @@ void innerPerformAction(String followerIndex, ClusterState currentClusterState,
             return;
         }
 
-        PauseFollowAction.Request request = new PauseFollowAction.Request(followerIndex);
-        request.masterNodeTimeout(TimeValue.MAX_VALUE);
+        PauseFollowAction.Request request = new PauseFollowAction.Request(TimeValue.MAX_VALUE, followerIndex);
         getClient().execute(PauseFollowAction.INSTANCE, request, listener.delegateFailureAndWrap((l, r) -> {
             if (r.isAcknowledged() == false) {
                 throw new ElasticsearchException("pause follow request failed to be acknowledged");
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStep.java
index 8b40a23cc3c44..6e07d4e6ac823 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStep.java
@@ -33,7 +33,7 @@ public boolean isRetryable() {
 
     @Override
     void innerPerformAction(String followerIndex, ClusterState currentClusterState, ActionListener<Void> listener) {
-        UnfollowAction.Request request = new UnfollowAction.Request(followerIndex).masterNodeTimeout(TimeValue.MAX_VALUE);
+        final var request = new UnfollowAction.Request(TimeValue.MAX_VALUE, TimeValue.MAX_VALUE, followerIndex);
         getClient().execute(UnfollowAction.INSTANCE, request, ActionListener.wrap(r -> {
             if (r.isAcknowledged() == false) {
                 throw new ElasticsearchException("unfollow request failed to be acknowledged");
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java
index fab9f8de5339f..7c1e71139cb0c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java
@@ -24,7 +24,7 @@ protected Writeable.Reader<CcrStatsAction.Request> instanceReader() {
 
     @Override
     protected CcrStatsAction.Request createTestInstance() {
-        var request = new CcrStatsAction.Request();
+        var request = new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT);
         request.setTimeout(TimeValue.timeValueSeconds(randomFrom(1, 5, 10, 15)));
         request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(1, 5, 10, 15)));
         return request;
@@ -34,13 +34,13 @@ protected CcrStatsAction.Request createTestInstance() {
     protected CcrStatsAction.Request mutateInstance(CcrStatsAction.Request instance) throws IOException {
         return switch (randomInt(1)) {
             case 0 -> {
-                var mutatedInstance = new CcrStatsAction.Request();
+                var mutatedInstance = new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT);
                 mutatedInstance.setTimeout(instance.getTimeout());
                 mutatedInstance.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30)));
                 yield mutatedInstance;
             }
             case 1 -> {
-                var mutatedInstance = new CcrStatsAction.Request();
+                var mutatedInstance = new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT);
                 mutatedInstance.setTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30)));
                 mutatedInstance.masterNodeTimeout(instance.masterNodeTimeout());
                 yield mutatedInstance;
@@ -51,7 +51,7 @@ protected CcrStatsAction.Request mutateInstance(CcrStatsAction.Request instance)
 
     public void testSerializationBwc() throws IOException {
         // In previous version `timeout` is not set
-        var request = new CcrStatsAction.Request();
+        var request = new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT);
         if (randomBoolean()) {
             request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30)));
         }
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java
index 9691756285f76..514540a85eb3b 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java
@@ -78,7 +78,7 @@ protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node, fin
             final long timestamp = timestamp();
             final String clusterUuid = clusterUuid(clusterState);
 
-            final CcrStatsAction.Request request = new CcrStatsAction.Request();
+            final CcrStatsAction.Request request = new CcrStatsAction.Request(getCollectionTimeout());
             final CcrStatsAction.Response response = client.execute(CcrStatsAction.INSTANCE, request).actionGet(getCollectionTimeout());
 
             final AutoFollowStatsMonitoringDoc autoFollowStatsDoc = new AutoFollowStatsMonitoringDoc(

From bdaaa9b838e5529f32de2261fec68102d6436769 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 May 2024 07:30:05 -0400
Subject: [PATCH 029/208] ESQL: Drop more "nested" query support (#109130)

This was all inherited from QL, but ESQL doesn't support `nested` fields
yet. And, when it does, it's not going to support nested fields in the
same way as QL. And, finally, none of these methods for nested support
weren't called by ESQL at all. So they were only used by a few tests.
---
 .../esql/core/querydsl/query/BoolQuery.java   |  31 ----
 .../esql/core/querydsl/query/ExistsQuery.java |   2 +-
 .../core/querydsl/query/GeoDistanceQuery.java |   2 +-
 .../esql/core/querydsl/query/LeafQuery.java   |  33 ----
 .../esql/core/querydsl/query/MatchAll.java    |   2 +-
 .../esql/core/querydsl/query/MatchQuery.java  |   2 +-
 .../core/querydsl/query/MultiMatchQuery.java  |   2 +-
 .../esql/core/querydsl/query/NestedQuery.java | 164 ------------------
 .../esql/core/querydsl/query/NotQuery.java    |  20 ---
 .../esql/core/querydsl/query/PrefixQuery.java |   2 +-
 .../xpack/esql/core/querydsl/query/Query.java |  22 ---
 .../core/querydsl/query/QueryStringQuery.java |   2 +-
 .../esql/core/querydsl/query/RangeQuery.java  |   2 +-
 .../esql/core/querydsl/query/RegexQuery.java  |   2 +-
 .../esql/core/querydsl/query/TermQuery.java   |   2 +-
 .../esql/core/querydsl/query/TermsQuery.java  |   2 +-
 .../core/querydsl/query/WildcardQuery.java    |   2 +-
 .../core/querydsl/query/BoolQueryTests.java   |  90 ++--------
 .../core/querydsl/query/LeafQueryTests.java   |  23 +--
 .../core/querydsl/query/NestedQueryTests.java | 143 ---------------
 .../esql/querydsl/query/SingleValueQuery.java |  16 --
 .../querydsl/query/SpatialRelatesQuery.java   |  16 --
 22 files changed, 27 insertions(+), 555 deletions(-)
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQuery.java
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQuery.java
 delete mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQueryTests.java

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java
index 3c0723b62d711..dbd75c93ee0e7 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java
@@ -8,12 +8,10 @@
 
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
@@ -44,35 +42,6 @@ public BoolQuery(Source source, boolean isAnd, List<Query> queries) {
         this.queries = queries;
     }
 
-    @Override
-    public boolean containsNestedField(String path, String field) {
-        for (Query query : queries) {
-            if (query.containsNestedField(path, field)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public Query addNestedField(String path, String field, String format, boolean hasDocValues) {
-        boolean unchanged = true;
-        List<Query> rewritten = new ArrayList<>(queries.size());
-        for (Query query : queries) {
-            var rewrittenQuery = query.addNestedField(path, field, format, hasDocValues);
-            unchanged &= rewrittenQuery == query;
-            rewritten.add(rewrittenQuery);
-        }
-        return unchanged ? this : new BoolQuery(source(), isAnd, rewritten);
-    }
-
-    @Override
-    public void enrichNestedSort(NestedSortBuilder sort) {
-        for (Query query : queries) {
-            query.enrichNestedSort(sort);
-        }
-    }
-
     @Override
     public QueryBuilder asBuilder() {
         BoolQueryBuilder boolQuery = boolQuery();
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java
index 7633f7f3f1e9c..be585232cf8d6 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java
@@ -11,7 +11,7 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
 
-public class ExistsQuery extends LeafQuery {
+public class ExistsQuery extends Query {
 
     private final String name;
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java
index f3c8829c2159b..f7843cec0c88c 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java
@@ -13,7 +13,7 @@
 
 import java.util.Objects;
 
-public class GeoDistanceQuery extends LeafQuery {
+public class GeoDistanceQuery extends Query {
 
     private final String field;
     private final double lat;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQuery.java
deleted file mode 100644
index 786114a52c7d5..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQuery.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.querydsl.query;
-
-import org.elasticsearch.search.sort.NestedSortBuilder;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-
-abstract class LeafQuery extends Query {
-    LeafQuery(Source source) {
-        super(source);
-    }
-
-    @Override
-    public final boolean containsNestedField(String path, String field) {
-        // No leaf queries are nested
-        return false;
-    }
-
-    @Override
-    public Query addNestedField(String path, String field, String format, boolean hasDocValues) {
-        // No leaf queries are nested
-        return this;
-    }
-
-    @Override
-    public void enrichNestedSort(NestedSortBuilder sort) {
-        // No leaf queries are nested
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java
index 13f624a283b16..6415a69e2201d 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java
@@ -11,7 +11,7 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 
-public class MatchAll extends LeafQuery {
+public class MatchAll extends Query {
     public MatchAll(Source source) {
         super(source);
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java
index 8c9934af9dfa6..3b7948d37cfad 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java
@@ -22,7 +22,7 @@
 
 import static java.util.Map.entry;
 
-public class MatchQuery extends LeafQuery {
+public class MatchQuery extends Query {
 
     private static final Map<String, BiConsumer<MatchQueryBuilder, String>> BUILDER_APPLIERS;
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java
index 8bede4f16d00e..71e3cb9fd494a 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java
@@ -21,7 +21,7 @@
 
 import static java.util.Map.entry;
 
-public class MultiMatchQuery extends LeafQuery {
+public class MultiMatchQuery extends Query {
 
     private static final Map<String, BiConsumer<MultiMatchQueryBuilder, String>> BUILDER_APPLIERS;
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQuery.java
deleted file mode 100644
index 65c953cdf818d..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQuery.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.querydsl.query;
-
-import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.index.query.InnerHitBuilder;
-import org.elasticsearch.index.query.NestedQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.fetch.StoredFieldsContext;
-import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
-import org.elasticsearch.search.sort.NestedSortBuilder;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-
-import java.util.AbstractMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonList;
-import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
-
-/**
- * A query to a nested document.
- */
-public class NestedQuery extends Query {
-    private static long COUNTER = 0;
-    // TODO: make this configurable
-    private static final int MAX_INNER_HITS = 99;
-    private static final List<String> NO_STORED_FIELD = singletonList(StoredFieldsContext._NONE_);
-
-    private final String path;
-    private final Map<String, Map.Entry<Boolean, String>> fields; // field -> (useDocValues, format)
-    private final Query child;
-
-    public NestedQuery(Source source, String path, Query child) {
-        this(source, path, emptyMap(), child);
-    }
-
-    public NestedQuery(Source source, String path, Map<String, Map.Entry<Boolean, String>> fields, Query child) {
-        super(source);
-        if (path == null) {
-            throw new IllegalArgumentException("path is required");
-        }
-        if (fields == null) {
-            throw new IllegalArgumentException("fields is required");
-        }
-        if (child == null) {
-            throw new IllegalArgumentException("child is required");
-        }
-        this.path = path;
-        this.fields = fields;
-        this.child = child;
-    }
-
-    @Override
-    public boolean containsNestedField(String otherPath, String field) {
-        boolean iContainThisField = this.path.equals(otherPath) && fields.containsKey(field);
-        boolean myChildContainsThisField = child.containsNestedField(otherPath, field);
-        return iContainThisField || myChildContainsThisField;
-    }
-
-    @Override
-    public Query addNestedField(String otherPath, String field, String format, boolean hasDocValues) {
-        if (false == this.path.equals(otherPath)) {
-            // I'm not at the right path so let my child query have a crack at it
-            Query rewrittenChild = child.addNestedField(otherPath, field, format, hasDocValues);
-            if (rewrittenChild == child) {
-                return this;
-            }
-            return new NestedQuery(source(), otherPath, fields, rewrittenChild);
-        }
-        if (fields.containsKey(field)) {
-            // I already have the field, no rewriting needed
-            return this;
-        }
-        Map<String, Map.Entry<Boolean, String>> newFields = Maps.newMapWithExpectedSize(fields.size() + 1);
-        newFields.putAll(fields);
-        newFields.put(field, new AbstractMap.SimpleImmutableEntry<>(hasDocValues, format));
-        return new NestedQuery(source(), otherPath, unmodifiableMap(newFields), child);
-    }
-
-    @Override
-    public void enrichNestedSort(NestedSortBuilder sort) {
-        child.enrichNestedSort(sort);
-        if (false == sort.getPath().equals(path)) {
-            return;
-        }
-
-        // TODO: Add all filters in nested sorting when https://github.com/elastic/elasticsearch/issues/33079 is implemented
-        // Adding multiple filters to sort sections makes sense for nested queries where multiple conditions belong to the same
-        // nested query. The current functionality creates one nested query for each condition involving a nested field.
-        QueryBuilder childAsBuilder = child.asBuilder();
-        if (sort.getFilter() != null && false == sort.getFilter().equals(childAsBuilder)) {
-            // throw new SqlIllegalArgumentException("nested query should have been grouped in one place");
-            return;
-        }
-        sort.setFilter(childAsBuilder);
-    }
-
-    @Override
-    public QueryBuilder asBuilder() {
-        // disable score
-        NestedQueryBuilder query = nestedQuery(path, child.asBuilder(), ScoreMode.None);
-
-        if (fields.isEmpty() == false) {
-            InnerHitBuilder ihb = new InnerHitBuilder();
-            ihb.setSize(0);
-            ihb.setSize(MAX_INNER_HITS);
-            ihb.setName(path + "_" + COUNTER++);
-
-            for (Map.Entry<String, Map.Entry<Boolean, String>> entry : fields.entrySet()) {
-                if (entry.getValue().getKey()) {
-                    ihb.addFetchField(entry.getKey(), entry.getValue().getValue());
-                } else {
-                    ihb.addFetchField(entry.getKey());
-                }
-            }
-            ihb.setFetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE);
-            ihb.setStoredFieldNames(NO_STORED_FIELD);
-
-            query.innerHit(ihb);
-        }
-
-        return query;
-    }
-
-    String path() {
-        return path;
-    }
-
-    Map<String, Map.Entry<Boolean, String>> fields() {
-        return fields;
-    }
-
-    Query child() {
-        return child;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(super.hashCode(), path, fields, child);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (false == super.equals(obj)) {
-            return false;
-        }
-        NestedQuery other = (NestedQuery) obj;
-        return path.equals(other.path) && fields.equals(other.fields) && child.equals(other.child);
-    }
-
-    @Override
-    protected String innerToString() {
-        return path + "." + fields + "[" + child + "]";
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java
index 4b87bbed80d71..4e36a4ee9f053 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.esql.core.querydsl.query;
 
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
 import java.util.Objects;
@@ -32,25 +31,6 @@ public Query child() {
         return child;
     }
 
-    @Override
-    public boolean containsNestedField(String path, String field) {
-        return child.containsNestedField(path, field);
-    }
-
-    @Override
-    public Query addNestedField(String path, String field, String format, boolean hasDocValues) {
-        Query rewrittenChild = child.addNestedField(path, field, format, hasDocValues);
-        if (child == rewrittenChild) {
-            return this;
-        }
-        return new NotQuery(source(), child);
-    }
-
-    @Override
-    public void enrichNestedSort(NestedSortBuilder sort) {
-        child.enrichNestedSort(sort);
-    }
-
     @Override
     public QueryBuilder asBuilder() {
         return boolQuery().mustNot(child.asBuilder());
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java
index 65dd014e72b70..1d98ff53be2f2 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java
@@ -13,7 +13,7 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
 
-public class PrefixQuery extends LeafQuery {
+public class PrefixQuery extends Query {
 
     private final String field, query;
     private final boolean caseInsensitive;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java
index 19db6aa79846a..f3154eb6cd377 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.esql.core.querydsl.query;
 
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
 /**
@@ -45,27 +44,6 @@ public Source source() {
         return source;
     }
 
-    /**
-     * Does this query contain a particular nested field?
-     */
-    public abstract boolean containsNestedField(String path, String field);
-
-    /**
-     * Rewrite this query to one that contains the specified nested field.
-     * <p>
-     * Used to make sure that we fetch nested fields even if they aren't
-     * explicitly part of the query.
-     * @return a new query if we could add the nested field, the same query
-     *      instance otherwise
-     */
-    public abstract Query addNestedField(String path, String field, String format, boolean hasDocValues);
-
-    /**
-     * Attach the one and only one matching nested query's filter to this
-     * sort.
-     */
-    public abstract void enrichNestedSort(NestedSortBuilder sort);
-
     /**
      * Convert to an Elasticsearch {@link QueryBuilder} all set up to execute
      * the query.
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java
index 32cd661cab579..8ac90e6314174 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java
@@ -24,7 +24,7 @@
 
 import static java.util.Map.entry;
 
-public class QueryStringQuery extends LeafQuery {
+public class QueryStringQuery extends Query {
 
     // TODO: it'd be great if these could be constants instead of Strings, needs a core change to make the fields public first
     private static final Map<String, BiConsumer<QueryStringQueryBuilder, String>> BUILDER_APPLIERS = Map.ofEntries(
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java
index 745e24fe52fc7..2d66ee86d0f61 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java
@@ -16,7 +16,7 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
 
-public class RangeQuery extends LeafQuery {
+public class RangeQuery extends Query {
 
     private final String field;
     private final Object lower, upper;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java
index 9117c32723d76..a8e48de654196 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java
@@ -13,7 +13,7 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.regexpQuery;
 
-public class RegexQuery extends LeafQuery {
+public class RegexQuery extends Query {
 
     private final String field, regex;
     private final boolean caseInsensitive;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java
index b195be3741ad1..240f9f581b27e 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java
@@ -14,7 +14,7 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 
-public class TermQuery extends LeafQuery {
+public class TermQuery extends Query {
 
     private final String term;
     private final Object value;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java
index a35a41ef09aa4..5b0920929853a 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java
@@ -14,7 +14,7 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
 
-public class TermsQuery extends LeafQuery {
+public class TermsQuery extends Query {
 
     private final String term;
     private final Set<Object> values;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java
index be2c771ced815..9266f2b43d081 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java
@@ -14,7 +14,7 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery;
 
-public class WildcardQuery extends LeafQuery {
+public class WildcardQuery extends Query {
 
     private final String field, query;
     private final boolean caseInsensitive;
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java
index c2f517aba4e1b..1c9d6bc54aebf 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java
@@ -6,18 +6,17 @@
  */
 package org.elasticsearch.xpack.esql.core.querydsl.query;
 
-import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.tree.SourceTests;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
-import java.util.AbstractMap.SimpleImmutableEntry;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -25,12 +24,16 @@
 
 public class BoolQueryTests extends ESTestCase {
     static BoolQuery randomBoolQuery(int depth) {
-        return new BoolQuery(
-            SourceTests.randomSource(),
-            randomBoolean(),
-            NestedQueryTests.randomQuery(depth),
-            NestedQueryTests.randomQuery(depth)
-        );
+        return new BoolQuery(SourceTests.randomSource(), randomBoolean(), randomQuery(depth), randomQuery(depth));
+    }
+
+    static Query randomQuery(int depth) {
+        List<Supplier<Query>> options = new ArrayList<>();
+        options.add(MatchQueryTests::randomMatchQuery);
+        if (depth > 0) {
+            options.add(() -> BoolQueryTests.randomBoolQuery(depth - 1));
+        }
+        return randomFrom(options).get();
     }
 
     public void testEqualsAndHashCode() {
@@ -45,77 +48,12 @@ private static BoolQuery mutate(BoolQuery query) {
         List<Function<BoolQuery, BoolQuery>> options = Arrays.asList(
             q -> new BoolQuery(SourceTests.mutate(q.source()), q.isAnd(), left(q), right(q)),
             q -> new BoolQuery(q.source(), false == q.isAnd(), left(q), right(q)),
-            q -> new BoolQuery(q.source(), q.isAnd(), randomValueOtherThan(left(q), () -> NestedQueryTests.randomQuery(5)), right(q)),
-            q -> new BoolQuery(q.source(), q.isAnd(), left(q), randomValueOtherThan(right(q), () -> NestedQueryTests.randomQuery(5)))
+            q -> new BoolQuery(q.source(), q.isAnd(), randomValueOtherThan(left(q), () -> randomQuery(5)), right(q)),
+            q -> new BoolQuery(q.source(), q.isAnd(), left(q), randomValueOtherThan(right(q), () -> randomQuery(5)))
         );
         return randomFrom(options).apply(query);
     }
 
-    public void testContainsNestedField() {
-        assertFalse(boolQueryWithoutNestedChildren().containsNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5)));
-
-        String path = randomAlphaOfLength(5);
-        String field = randomAlphaOfLength(5);
-        assertTrue(boolQueryWithNestedChildren(path, field).containsNestedField(path, field));
-    }
-
-    public void testAddNestedField() {
-        Query q = boolQueryWithoutNestedChildren();
-        assertSame(q, q.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), null, randomBoolean()));
-
-        String path = randomAlphaOfLength(5);
-        String field = randomAlphaOfLength(5);
-        q = boolQueryWithNestedChildren(path, field);
-        String newField = randomAlphaOfLength(5);
-        boolean hasDocValues = randomBoolean();
-        Query rewritten = q.addNestedField(path, newField, null, hasDocValues);
-        assertNotSame(q, rewritten);
-        assertTrue(rewritten.containsNestedField(path, newField));
-    }
-
-    public void testEnrichNestedSort() {
-        Query q = boolQueryWithoutNestedChildren();
-        NestedSortBuilder sort = new NestedSortBuilder(randomAlphaOfLength(5));
-        q.enrichNestedSort(sort);
-        assertNull(sort.getFilter());
-
-        String path = randomAlphaOfLength(5);
-        String field = randomAlphaOfLength(5);
-        q = boolQueryWithNestedChildren(path, field);
-        sort = new NestedSortBuilder(path);
-        q.enrichNestedSort(sort);
-        assertNotNull(sort.getFilter());
-    }
-
-    private Query boolQueryWithoutNestedChildren() {
-        return new BoolQuery(
-            SourceTests.randomSource(),
-            randomBoolean(),
-            new MatchAll(SourceTests.randomSource()),
-            new MatchAll(SourceTests.randomSource())
-        );
-    }
-
-    private Query boolQueryWithNestedChildren(String path, String field) {
-        NestedQuery match = new NestedQuery(
-            SourceTests.randomSource(),
-            path,
-            singletonMap(field, new SimpleImmutableEntry<>(randomBoolean(), null)),
-            new MatchAll(SourceTests.randomSource())
-        );
-        Query matchAll = new MatchAll(SourceTests.randomSource());
-        Query left;
-        Query right;
-        if (randomBoolean()) {
-            left = match;
-            right = matchAll;
-        } else {
-            left = matchAll;
-            right = match;
-        }
-        return new BoolQuery(SourceTests.randomSource(), randomBoolean(), left, right);
-    }
-
     public void testToString() {
         assertEquals(
             "BoolQuery@1:2[ExistsQuery@1:2[f1] AND ExistsQuery@1:8[f2]]",
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java
index 1ac96b6d30e8c..15c49f58572cb 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.esql.core.querydsl.query;
 
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.tree.Location;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -18,7 +17,7 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class LeafQueryTests extends ESTestCase {
-    private static class DummyLeafQuery extends LeafQuery {
+    private static class DummyLeafQuery extends Query {
         private DummyLeafQuery(Source source) {
             super(source);
         }
@@ -47,26 +46,6 @@ private static DummyLeafQuery mutate(DummyLeafQuery query) {
         return new DummyLeafQuery(SourceTests.mutate(query.source()));
     }
 
-    public void testContainsNestedField() {
-        Query query = new DummyLeafQuery(SourceTests.randomSource());
-        // Leaf queries don't contain nested fields.
-        assertFalse(query.containsNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5)));
-    }
-
-    public void testAddNestedField() {
-        Query query = new DummyLeafQuery(SourceTests.randomSource());
-        // Leaf queries don't contain nested fields.
-        assertSame(query, query.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), null, randomBoolean()));
-    }
-
-    public void testEnrichNestedSort() {
-        Query query = new DummyLeafQuery(SourceTests.randomSource());
-        // Leaf queries don't contain nested fields.
-        NestedSortBuilder sort = new NestedSortBuilder(randomAlphaOfLength(5));
-        query.enrichNestedSort(sort);
-        assertNull(sort.getFilter());
-    }
-
     public void testNot() {
         var q = new LeafQueryTests.DummyLeafQuery(new Source(Location.EMPTY, "test"));
         assertThat(q.negate(new Source(Location.EMPTY, "not")), equalTo(new NotQuery(new Source(Location.EMPTY, "not"), q)));
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQueryTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQueryTests.java
deleted file mode 100644
index 1a36531efec52..0000000000000
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQueryTests.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.querydsl.query;
-
-import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.search.sort.NestedSortBuilder;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.tree.SourceTests;
-import org.elasticsearch.xpack.esql.core.util.StringUtils;
-
-import java.util.AbstractMap.SimpleImmutableEntry;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.function.Supplier;
-
-import static java.util.Collections.singletonMap;
-import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
-import static org.hamcrest.Matchers.hasEntry;
-
-public class NestedQueryTests extends ESTestCase {
-    static Query randomQuery(int depth) {
-        List<Supplier<Query>> options = new ArrayList<>();
-        options.add(MatchQueryTests::randomMatchQuery);
-        if (depth > 0) {
-            options.add(() -> randomNestedQuery(depth - 1));
-            options.add(() -> BoolQueryTests.randomBoolQuery(depth - 1));
-        }
-        return randomFrom(options).get();
-    }
-
-    static NestedQuery randomNestedQuery(int depth) {
-        return new NestedQuery(SourceTests.randomSource(), randomAlphaOfLength(5), randomFields(), randomQuery(depth));
-    }
-
-    private static Map<String, Map.Entry<Boolean, String>> randomFields() {
-        int size = between(0, 5);
-        Map<String, Map.Entry<Boolean, String>> fields = Maps.newMapWithExpectedSize(size);
-        while (fields.size() < size) {
-            fields.put(randomAlphaOfLength(5), new SimpleImmutableEntry<>(randomBoolean(), null));
-        }
-        return fields;
-    }
-
-    public void testEqualsAndHashCode() {
-        checkEqualsAndHashCode(randomNestedQuery(5), NestedQueryTests::copy, NestedQueryTests::mutate);
-    }
-
-    private static NestedQuery copy(NestedQuery query) {
-        return new NestedQuery(query.source(), query.path(), query.fields(), query.child());
-    }
-
-    private static NestedQuery mutate(NestedQuery query) {
-        List<Function<NestedQuery, NestedQuery>> options = Arrays.asList(
-            q -> new NestedQuery(SourceTests.mutate(q.source()), q.path(), q.fields(), q.child()),
-            q -> new NestedQuery(q.source(), randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5)), q.fields(), q.child()),
-            q -> new NestedQuery(q.source(), q.path(), randomValueOtherThan(q.fields(), NestedQueryTests::randomFields), q.child()),
-            q -> new NestedQuery(q.source(), q.path(), q.fields(), randomValueOtherThan(q.child(), () -> randomQuery(5)))
-        );
-        return randomFrom(options).apply(query);
-    }
-
-    public void testContainsNestedField() {
-        NestedQuery q = randomNestedQuery(0);
-        for (String field : q.fields().keySet()) {
-            assertTrue(q.containsNestedField(q.path(), field));
-            assertFalse(q.containsNestedField(randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5)), field));
-        }
-        assertFalse(q.containsNestedField(q.path(), randomValueOtherThanMany(q.fields()::containsKey, () -> randomAlphaOfLength(5))));
-    }
-
-    public void testAddNestedField() {
-        NestedQuery q = randomNestedQuery(0);
-        for (String field : q.fields().keySet()) {
-            // add does nothing if the field is already there
-            assertSame(q, q.addNestedField(q.path(), field, null, randomBoolean()));
-            String otherPath = randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5));
-            // add does nothing if the path doesn't match
-            assertSame(q, q.addNestedField(otherPath, randomAlphaOfLength(5), null, randomBoolean()));
-        }
-
-        // if the field isn't in the list then add rewrites to a query with all the old fields and the new one
-        String newField = randomValueOtherThanMany(q.fields()::containsKey, () -> randomAlphaOfLength(5));
-        boolean hasDocValues = randomBoolean();
-        NestedQuery added = (NestedQuery) q.addNestedField(q.path(), newField, null, hasDocValues);
-        assertNotSame(q, added);
-        assertThat(added.fields(), hasEntry(newField, new SimpleImmutableEntry<>(hasDocValues, null)));
-        assertTrue(added.containsNestedField(q.path(), newField));
-        for (Map.Entry<String, Map.Entry<Boolean, String>> field : q.fields().entrySet()) {
-            assertThat(added.fields(), hasEntry(field.getKey(), field.getValue()));
-            assertTrue(added.containsNestedField(q.path(), field.getKey()));
-        }
-    }
-
-    public void testEnrichNestedSort() {
-        NestedQuery q = randomNestedQuery(0);
-
-        // enrich adds the filter if the path matches
-        {
-            NestedSortBuilder sort = new NestedSortBuilder(q.path());
-            q.enrichNestedSort(sort);
-            assertEquals(q.child().asBuilder(), sort.getFilter());
-        }
-
-        // but doesn't if it doesn't match
-        {
-            NestedSortBuilder sort = new NestedSortBuilder(randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5)));
-            q.enrichNestedSort(sort);
-            assertNull(sort.getFilter());
-        }
-
-        // enriching with the same query twice is fine
-        {
-            NestedSortBuilder sort = new NestedSortBuilder(q.path());
-            q.enrichNestedSort(sort);
-            assertEquals(q.child().asBuilder(), sort.getFilter());
-            q.enrichNestedSort(sort);
-
-            // But enriching using another query will keep only the first query
-            Query originalChildQuery = randomValueOtherThan(q.child(), () -> randomQuery(0));
-            NestedQuery other = new NestedQuery(SourceTests.randomSource(), q.path(), q.fields(), originalChildQuery);
-            other.enrichNestedSort(sort);
-            assertEquals(other.child().asBuilder(), originalChildQuery.asBuilder());
-        }
-    }
-
-    public void testToString() {
-        NestedQuery q = new NestedQuery(
-            new Source(1, 1, StringUtils.EMPTY),
-            "a.b",
-            singletonMap("f", new SimpleImmutableEntry<>(true, null)),
-            new MatchAll(new Source(1, 1, StringUtils.EMPTY))
-        );
-        assertEquals("NestedQuery@1:2[a.b.{f=true=null}[MatchAll@1:2[]]]", q.toString());
-    }
-}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java
index 8dd1cbdef1386..6545e892741d2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
-import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -83,21 +82,6 @@ public SingleValueQuery(Query next, String field) {
         this.field = field;
     }
 
-    @Override
-    public boolean containsNestedField(String path, String field) {
-        return next.containsNestedField(path, field);
-    }
-
-    @Override
-    public Query addNestedField(String path, String field, String format, boolean hasDocValues) {
-        return next.addNestedField(path, field, format, hasDocValues);
-    }
-
-    @Override
-    public void enrichNestedSort(NestedSortBuilder sort) {
-        next.enrichNestedSort(sort);
-    }
-
     @Override
     public Builder asBuilder() {
         return new Builder(next.asBuilder(), field, new Stats(), next.source());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
index c7875e79533a7..f02dd63999e27 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.lucene.spatial.CartesianShapeDocValuesQuery;
-import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -57,21 +56,6 @@ public SpatialRelatesQuery(Source source, String field, ShapeField.QueryRelation
         this.dataType = dataType;
     }
 
-    @Override
-    public boolean containsNestedField(String path, String field) {
-        return false;
-    }
-
-    @Override
-    public Query addNestedField(String path, String field, String format, boolean hasDocValues) {
-        return null;
-    }
-
-    @Override
-    public void enrichNestedSort(NestedSortBuilder sort) {
-
-    }
-
     @Override
     public QueryBuilder asBuilder() {
         return EsqlDataTypes.isSpatialGeo(dataType) ? new GeoShapeQueryBuilder() : new CartesianShapeQueryBuilder();

From b6241711ef49deeaa2d550bd34d4838d0414301c Mon Sep 17 00:00:00 2001
From: Liam Thompson <32779855+leemthompo@users.noreply.github.com>
Date: Wed, 29 May 2024 14:00:19 +0200
Subject: [PATCH 030/208] [DOCS] Update CCS matrix for 8.14 (#109142)

---
 .../ccs-version-compat-matrix.asciidoc        | 43 ++++++++++---------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
index c1d279d3163e8..4a5efe09ea5a0 100644
--- a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
+++ b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
@@ -1,23 +1,24 @@
-[cols="^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^"]
+[cols="^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^"]
 |====
-| 17+^h| Remote cluster version
+| 18+^h| Remote cluster version
 h| Local cluster version
-            |  6.8        | 7.1–7.16   | 7.17       | 8.0        | 8.1        | 8.2        | 8.3       | 8.4       | 8.5       |8.6         |8.7         |8.8         |8.9         |8.10        |8.11       |8.12 | 8.13
-| 6.8       |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon}
-| 7.1–7.16  |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon}
-| 7.17      |  {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}  | {yes-icon}
-| 8.0       |  {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.1       |  {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.2       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.3       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}|{yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.4       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} |{yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.5       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  |{yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.6       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.7       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.8       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.9       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.10      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.11      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.12      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon}| {yes-icon} | {yes-icon}
-| 8.13      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} |  {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {yes-icon} | {yes-icon} 
-|====
\ No newline at end of file
+            |  6.8        | 7.1–7.16   | 7.17       | 8.0        | 8.1        | 8.2        | 8.3       | 8.4       | 8.5       | 8.6        | 8.7        | 8.8        | 8.9        | 8.10       | 8.11       | 8.12      | 8.13      | 8.14
+| 6.8       |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}
+| 7.1–7.16  |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}
+| 7.17      |  {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.0       |  {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.1       |  {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.2       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.3       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.4       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.5       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.6       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.7       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.8       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.9       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.10      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.11      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.12      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.13      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.14      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}
+|====

From 849c33edeb638efb59860fe1795dc543fa7073b7 Mon Sep 17 00:00:00 2001
From: Kathleen DeRusso <kathleen.derusso@elastic.co>
Date: Wed, 29 May 2024 08:02:05 -0400
Subject: [PATCH 031/208] Add SearchOrgs to changelog schema (#109135)

---
 .../src/main/resources/changelog-schema.json     | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json
index 2a0cda7fa33c9..8bdfb8d6c0b9c 100644
--- a/build-tools-internal/src/main/resources/changelog-schema.json
+++ b/build-tools-internal/src/main/resources/changelog-schema.json
@@ -39,6 +39,8 @@
             "EQL",
             "ES|QL",
             "Engine",
+            "Experiences",
+            "Extract&Transform",
             "FIPS",
             "Features",
             "Geo",
@@ -48,6 +50,7 @@
             "ILM+SLM",
             "IdentityProvider",
             "Indices APIs",
+            "Inference",
             "Infra/CLI",
             "Infra/Circuit Breakers",
             "Infra/Core",
@@ -76,6 +79,7 @@
             "Ranking",
             "Recovery",
             "Reindex",
+            "Relevance",
             "Rollup",
             "SQL",
             "Search",
@@ -174,7 +178,9 @@
             }
           },
           "then": {
-            "required": ["breaking"]
+            "required": [
+              "breaking"
+            ]
           }
         },
         {
@@ -186,7 +192,9 @@
             }
           },
           "then": {
-            "required": ["breaking"]
+            "required": [
+              "breaking"
+            ]
           }
         }
       ],
@@ -198,7 +206,9 @@
         }
       },
       "then": {
-        "required": ["deprecation"]
+        "required": [
+          "deprecation"
+        ]
       },
       "additionalProperties": false
     },

From 1e58f3a48557e14c791a8f3c7c3d5329c9278cd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?=
 <istvan.szabo@elastic.co>
Date: Wed, 29 May 2024 14:56:59 +0200
Subject: [PATCH 032/208] [DOCS] Fixes sparse vector query docs. (#109153)

---
 docs/reference/query-dsl/sparse-vector-query.asciidoc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/reference/query-dsl/sparse-vector-query.asciidoc b/docs/reference/query-dsl/sparse-vector-query.asciidoc
index cb3f855afaa51..9a269ad9712a8 100644
--- a/docs/reference/query-dsl/sparse-vector-query.asciidoc
+++ b/docs/reference/query-dsl/sparse-vector-query.asciidoc
@@ -36,6 +36,7 @@ GET _search
 ----
 // TEST[skip: Requires inference]
 
+[discrete]
 === Example request using precomputed vectors
 
 [source,console]

From 86652bde14e00b330402b2ed5366c0f3fe27300a Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 May 2024 09:05:00 -0400
Subject: [PATCH 033/208] ESQL: Make `Source` `Writeable` (#109137)

We'd love to move all of the serialization from `PlanNamedTypes` into
the the types themselves. But serialization for `Source` is in ESQL
proper, not ESQL-core with all of the types that need to be serialized.
This moves that serialization. It does so by making an interface in
ESQL-core that contains the `query` itself which is required to
serialize `Source`. This seems like the simplest way to get started
moving serialization. The shim *should* be temporary, but it'll take
some time to move things around.
---
 .../xpack/esql/core/tree/Source.java          |  66 +++-
 .../xpack/esql/core/util/PlanStreamInput.java |  26 ++
 .../xpack/esql/core/util/SourceUtils.java     |  76 +---
 .../xpack/esql/io/stream/PlanNamedTypes.java  | 324 +++++++++---------
 .../xpack/esql/io/stream/PlanStreamInput.java |  17 +-
 .../esql/io/stream/PlanStreamOutput.java      |  12 -
 .../xpack/ql/util/SourceUtils.java            |  93 -----
 7 files changed, 283 insertions(+), 331 deletions(-)
 create mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
 delete mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/SourceUtils.java

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java
index 23d05bdf99273..2129bd8743d0f 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java
@@ -7,9 +7,17 @@
 
 package org.elasticsearch.xpack.esql.core.tree;
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
+import org.elasticsearch.xpack.esql.core.util.StringUtils;
+
+import java.io.IOException;
 import java.util.Objects;
 
-public final class Source {
+public final class Source implements Writeable {
 
     public static final Source EMPTY = new Source(Location.EMPTY, "");
 
@@ -25,6 +33,31 @@ public Source(Location location, String text) {
         this.text = text;
     }
 
+    public static <S extends StreamInput & PlanStreamInput> Source readFrom(S in) throws IOException {
+        if (in.readBoolean() == false) {
+            return EMPTY;
+        }
+        int line = in.readInt();
+        int column = in.readInt();
+        int charPositionInLine = column - 1;
+
+        int length = in.readInt();
+        String text = sourceText(in.sourceText(), line, column, length);
+        return new Source(new Location(line, charPositionInLine), text);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        if (this == EMPTY) {
+            out.writeBoolean(false);
+            return;
+        }
+        out.writeBoolean(true);
+        out.writeInt(location.getLineNumber());
+        out.writeInt(location.getColumnNumber());
+        out.writeInt(text.length());
+    }
+
     // TODO: rename to location()
     public Location source() {
         return location;
@@ -61,4 +94,35 @@ public String toString() {
     public static Source synthetic(String text) {
         return new Source(Location.EMPTY, text);
     }
+
+    private static String sourceText(String query, int line, int column, int length) {
+        if (line <= 0 || column <= 0 || query.isEmpty()) {
+            return StringUtils.EMPTY;
+        }
+        int offset = textOffset(query, line, column);
+        if (offset + length > query.length()) {
+            throw new QlIllegalArgumentException(
+                "location [@" + line + ":" + column + "] and length [" + length + "] overrun query size [" + query.length() + "]"
+            );
+        }
+        return query.substring(offset, offset + length);
+    }
+
+    private static int textOffset(String query, int line, int column) {
+        int offset = 0;
+        if (line > 1) {
+            String[] lines = query.split("\n");
+            if (line > lines.length) {
+                throw new QlIllegalArgumentException(
+                    "line location [" + line + "] higher than max [" + lines.length + "] in query [" + query + "]"
+                );
+            }
+            for (int i = 0; i < line - 1; i++) {
+                offset += lines[i].length() + 1; // +1 accounts for the removed \n
+            }
+        }
+        offset += column - 1; // -1 since column is 1-based indexed
+        return offset;
+    }
+
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
new file mode 100644
index 0000000000000..51898f36d5497
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.util;
+
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+/**
+ * Interface for streams that can serialize plan components. This exists so
+ * ESQL proper can expose streaming capability to ESQL-core. If the world is kind
+ * and just we'll remove this when we flatten everything from ESQL-core into
+ * ESQL proper.
+ */
+public interface PlanStreamInput {
+    /**
+     * The query sent by the user to build this plan. This is used to rebuild
+     * {@link Source} without sending the query over the wire over and over
+     * and over again.
+     */
+    String sourceText();
+
+}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java
index cbcb31c7a3b56..6bed7f06fa2f5 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java
@@ -9,8 +9,6 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.tree.Location;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
@@ -20,74 +18,30 @@ public final class SourceUtils {
 
     private SourceUtils() {}
 
+    /**
+     * Write a {@link Source} including the text in it.
+     * @deprecated replace with {@link Source#writeTo}.
+     *             That's not binary compatible so the replacement is complex.
+     */
+    @Deprecated
     public static void writeSource(StreamOutput out, Source source) throws IOException {
-        writeSource(out, source, true);
-    }
-
-    public static void writeSourceNoText(StreamOutput out, Source source) throws IOException {
-        writeSource(out, source, false);
-    }
-
-    public static Source readSource(StreamInput in) throws IOException {
-        return readSource(in, null);
-    }
-
-    public static Source readSourceWithText(StreamInput in, String queryText) throws IOException {
-        return readSource(in, queryText);
-    }
-
-    private static void writeSource(StreamOutput out, Source source, boolean writeText) throws IOException {
         out.writeInt(source.source().getLineNumber());
         out.writeInt(source.source().getColumnNumber());
-        if (writeText) {
-            out.writeString(source.text());
-        } else {
-            out.writeInt(source.text().length());
-        }
+        out.writeString(source.text());
     }
 
-    private static Source readSource(StreamInput in, @Nullable String queryText) throws IOException {
+    /**
+     * Read a {@link Source} including the text in it.
+     * @deprecated replace with {@link Source#readFrom(StreamInput)}.
+     *             That's not binary compatible so the replacement is complex.
+     */
+    @Deprecated
+    public static Source readSource(StreamInput in) throws IOException {
         int line = in.readInt();
         int column = in.readInt();
         int charPositionInLine = column - 1;
 
-        String text;
-        if (queryText == null) {
-            text = in.readString();
-        } else {
-            int length = in.readInt();
-            text = sourceText(queryText, line, column, length);
-        }
+        String text = in.readString();
         return new Source(new Location(line, charPositionInLine), text);
     }
-
-    private static String sourceText(String query, int line, int column, int length) {
-        if (line <= 0 || column <= 0 || query.isEmpty()) {
-            return StringUtils.EMPTY;
-        }
-        int offset = textOffset(query, line, column);
-        if (offset + length > query.length()) {
-            throw new QlIllegalArgumentException(
-                "location [@" + line + ":" + column + "] and length [" + length + "] overrun query size [" + query.length() + "]"
-            );
-        }
-        return query.substring(offset, offset + length);
-    }
-
-    private static int textOffset(String query, int line, int column) {
-        int offset = 0;
-        if (line > 1) {
-            String[] lines = query.split("\n");
-            if (line > lines.length) {
-                throw new QlIllegalArgumentException(
-                    "line location [" + line + "] higher than max [" + lines.length + "] in query [" + query + "]"
-                );
-            }
-            for (int i = 0; i < line - 1; i++) {
-                offset += lines[i].length() + 1; // +1 accounts for the removed \n
-            }
-        }
-        offset += column - 1; // -1 since column is 1-based indexed
-        return offset;
-    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index d64d70470c605..e846360258ebe 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -458,7 +458,7 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
     // -- physical plan nodes
     static AggregateExec readAggregateExec(PlanStreamInput in) throws IOException {
         return new AggregateExec(
-            in.readSource(),
+            Source.readFrom(in),
             in.readPhysicalPlanNode(),
             in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)),
             readNamedExpressions(in),
@@ -468,7 +468,7 @@ static AggregateExec readAggregateExec(PlanStreamInput in) throws IOException {
     }
 
     static void writeAggregateExec(PlanStreamOutput out, AggregateExec aggregateExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(aggregateExec.child());
         out.writeCollection(aggregateExec.groupings(), writerFromPlanWriter(PlanStreamOutput::writeExpression));
         writeNamedExpressions(out, aggregateExec.aggregates());
@@ -477,11 +477,17 @@ static void writeAggregateExec(PlanStreamOutput out, AggregateExec aggregateExec
     }
 
     static DissectExec readDissectExec(PlanStreamInput in) throws IOException {
-        return new DissectExec(in.readSource(), in.readPhysicalPlanNode(), in.readExpression(), readDissectParser(in), readAttributes(in));
+        return new DissectExec(
+            Source.readFrom(in),
+            in.readPhysicalPlanNode(),
+            in.readExpression(),
+            readDissectParser(in),
+            readAttributes(in)
+        );
     }
 
     static void writeDissectExec(PlanStreamOutput out, DissectExec dissectExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(dissectExec.child());
         out.writeExpression(dissectExec.inputExpression());
         writeDissectParser(out, dissectExec.parser());
@@ -490,7 +496,7 @@ static void writeDissectExec(PlanStreamOutput out, DissectExec dissectExec) thro
 
     static EsQueryExec readEsQueryExec(PlanStreamInput in) throws IOException {
         return new EsQueryExec(
-            in.readSource(),
+            Source.readFrom(in),
             readEsIndex(in),
             readIndexMode(in),
             readAttributes(in),
@@ -503,7 +509,7 @@ static EsQueryExec readEsQueryExec(PlanStreamInput in) throws IOException {
 
     static void writeEsQueryExec(PlanStreamOutput out, EsQueryExec esQueryExec) throws IOException {
         assert esQueryExec.children().size() == 0;
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         writeEsIndex(out, esQueryExec.index());
         writeIndexMode(out, esQueryExec.indexMode());
         writeAttributes(out, esQueryExec.output());
@@ -515,7 +521,7 @@ static void writeEsQueryExec(PlanStreamOutput out, EsQueryExec esQueryExec) thro
 
     static EsSourceExec readEsSourceExec(PlanStreamInput in) throws IOException {
         return new EsSourceExec(
-            in.readSource(),
+            Source.readFrom(in),
             readEsIndex(in),
             readAttributes(in),
             in.readOptionalNamedWriteable(QueryBuilder.class),
@@ -524,7 +530,7 @@ static EsSourceExec readEsSourceExec(PlanStreamInput in) throws IOException {
     }
 
     static void writeEsSourceExec(PlanStreamOutput out, EsSourceExec esSourceExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         writeEsIndex(out, esSourceExec.index());
         writeAttributes(out, esSourceExec.output());
         out.writeOptionalNamedWriteable(esSourceExec.query());
@@ -548,17 +554,17 @@ static void writeIndexMode(StreamOutput out, IndexMode indexMode) throws IOExcep
     }
 
     static EvalExec readEvalExec(PlanStreamInput in) throws IOException {
-        return new EvalExec(in.readSource(), in.readPhysicalPlanNode(), readAliases(in));
+        return new EvalExec(Source.readFrom(in), in.readPhysicalPlanNode(), readAliases(in));
     }
 
     static void writeEvalExec(PlanStreamOutput out, EvalExec evalExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(evalExec.child());
         writeAliases(out, evalExec.fields());
     }
 
     static EnrichExec readEnrichExec(PlanStreamInput in) throws IOException {
-        final Source source = in.readSource();
+        final Source source = Source.readFrom(in);
         final PhysicalPlan child = in.readPhysicalPlanNode();
         final NamedExpression matchField = in.readNamedExpression();
         final String policyName = in.readString();
@@ -593,7 +599,7 @@ static EnrichExec readEnrichExec(PlanStreamInput in) throws IOException {
     }
 
     static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(enrich.child());
         out.writeNamedExpression(enrich.matchField());
         out.writeString(enrich.policyName());
@@ -616,29 +622,29 @@ static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOEx
     }
 
     static ExchangeExec readExchangeExec(PlanStreamInput in) throws IOException {
-        return new ExchangeExec(in.readSource(), readAttributes(in), in.readBoolean(), in.readPhysicalPlanNode());
+        return new ExchangeExec(Source.readFrom(in), readAttributes(in), in.readBoolean(), in.readPhysicalPlanNode());
     }
 
     static void writeExchangeExec(PlanStreamOutput out, ExchangeExec exchangeExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         writeAttributes(out, exchangeExec.output());
         out.writeBoolean(exchangeExec.isInBetweenAggs());
         out.writePhysicalPlanNode(exchangeExec.child());
     }
 
     static ExchangeSinkExec readExchangeSinkExec(PlanStreamInput in) throws IOException {
-        return new ExchangeSinkExec(in.readSource(), readAttributes(in), in.readBoolean(), in.readPhysicalPlanNode());
+        return new ExchangeSinkExec(Source.readFrom(in), readAttributes(in), in.readBoolean(), in.readPhysicalPlanNode());
     }
 
     static void writeExchangeSinkExec(PlanStreamOutput out, ExchangeSinkExec exchangeSinkExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         writeAttributes(out, exchangeSinkExec.output());
         out.writeBoolean(exchangeSinkExec.isIntermediateAgg());
         out.writePhysicalPlanNode(exchangeSinkExec.child());
     }
 
     static ExchangeSourceExec readExchangeSourceExec(PlanStreamInput in) throws IOException {
-        return new ExchangeSourceExec(in.readSource(), readAttributes(in), in.readBoolean());
+        return new ExchangeSourceExec(Source.readFrom(in), readAttributes(in), in.readBoolean());
     }
 
     static void writeExchangeSourceExec(PlanStreamOutput out, ExchangeSourceExec exchangeSourceExec) throws IOException {
@@ -647,28 +653,28 @@ static void writeExchangeSourceExec(PlanStreamOutput out, ExchangeSourceExec exc
     }
 
     static FieldExtractExec readFieldExtractExec(PlanStreamInput in) throws IOException {
-        return new FieldExtractExec(in.readSource(), in.readPhysicalPlanNode(), readAttributes(in));
+        return new FieldExtractExec(Source.readFrom(in), in.readPhysicalPlanNode(), readAttributes(in));
     }
 
     static void writeFieldExtractExec(PlanStreamOutput out, FieldExtractExec fieldExtractExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(fieldExtractExec.child());
         writeAttributes(out, fieldExtractExec.attributesToExtract());
     }
 
     static FilterExec readFilterExec(PlanStreamInput in) throws IOException {
-        return new FilterExec(in.readSource(), in.readPhysicalPlanNode(), in.readExpression());
+        return new FilterExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readExpression());
     }
 
     static void writeFilterExec(PlanStreamOutput out, FilterExec filterExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(filterExec.child());
         out.writeExpression(filterExec.condition());
     }
 
     static FragmentExec readFragmentExec(PlanStreamInput in) throws IOException {
         return new FragmentExec(
-            in.readSource(),
+            Source.readFrom(in),
             in.readLogicalPlanNode(),
             in.readOptionalNamedWriteable(QueryBuilder.class),
             in.readOptionalVInt(),
@@ -677,7 +683,7 @@ static FragmentExec readFragmentExec(PlanStreamInput in) throws IOException {
     }
 
     static void writeFragmentExec(PlanStreamOutput out, FragmentExec fragmentExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(fragmentExec.fragment());
         out.writeOptionalNamedWriteable(fragmentExec.esFilter());
         out.writeOptionalVInt(fragmentExec.estimatedRowSize());
@@ -689,7 +695,7 @@ static void writeFragmentExec(PlanStreamOutput out, FragmentExec fragmentExec) t
     static GrokExec readGrokExec(PlanStreamInput in) throws IOException {
         Source source;
         return new GrokExec(
-            source = in.readSource(),
+            source = Source.readFrom(in),
             in.readPhysicalPlanNode(),
             in.readExpression(),
             Grok.pattern(source, in.readString()),
@@ -698,7 +704,7 @@ static GrokExec readGrokExec(PlanStreamInput in) throws IOException {
     }
 
     static void writeGrokExec(PlanStreamOutput out, GrokExec grokExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(grokExec.child());
         out.writeExpression(grokExec.inputExpression());
         out.writeString(grokExec.pattern().pattern());
@@ -706,21 +712,21 @@ static void writeGrokExec(PlanStreamOutput out, GrokExec grokExec) throws IOExce
     }
 
     static LimitExec readLimitExec(PlanStreamInput in) throws IOException {
-        return new LimitExec(in.readSource(), in.readPhysicalPlanNode(), in.readNamed(Expression.class));
+        return new LimitExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readNamed(Expression.class));
     }
 
     static void writeLimitExec(PlanStreamOutput out, LimitExec limitExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(limitExec.child());
         out.writeExpression(limitExec.limit());
     }
 
     static MvExpandExec readMvExpandExec(PlanStreamInput in) throws IOException {
-        return new MvExpandExec(in.readSource(), in.readPhysicalPlanNode(), in.readNamedExpression(), in.readAttribute());
+        return new MvExpandExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readNamedExpression(), in.readAttribute());
     }
 
     static void writeMvExpandExec(PlanStreamOutput out, MvExpandExec mvExpandExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(mvExpandExec.child());
         out.writeNamedExpression(mvExpandExec.target());
         out.writeAttribute(mvExpandExec.expanded());
@@ -728,52 +734,52 @@ static void writeMvExpandExec(PlanStreamOutput out, MvExpandExec mvExpandExec) t
 
     static OrderExec readOrderExec(PlanStreamInput in) throws IOException {
         return new OrderExec(
-            in.readSource(),
+            Source.readFrom(in),
             in.readPhysicalPlanNode(),
             in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder))
         );
     }
 
     static void writeOrderExec(PlanStreamOutput out, OrderExec orderExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(orderExec.child());
         out.writeCollection(orderExec.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder));
     }
 
     static ProjectExec readProjectExec(PlanStreamInput in) throws IOException {
-        return new ProjectExec(in.readSource(), in.readPhysicalPlanNode(), readNamedExpressions(in));
+        return new ProjectExec(Source.readFrom(in), in.readPhysicalPlanNode(), readNamedExpressions(in));
     }
 
     static void writeProjectExec(PlanStreamOutput out, ProjectExec projectExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(projectExec.child());
         writeNamedExpressions(out, projectExec.projections());
     }
 
     static RowExec readRowExec(PlanStreamInput in) throws IOException {
-        return new RowExec(in.readSource(), readAliases(in));
+        return new RowExec(Source.readFrom(in), readAliases(in));
     }
 
     static void writeRowExec(PlanStreamOutput out, RowExec rowExec) throws IOException {
         assert rowExec.children().size() == 0;
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         writeAliases(out, rowExec.fields());
     }
 
     @SuppressWarnings("unchecked")
     static ShowExec readShowExec(PlanStreamInput in) throws IOException {
-        return new ShowExec(in.readSource(), readAttributes(in), (List<List<Object>>) in.readGenericValue());
+        return new ShowExec(Source.readFrom(in), readAttributes(in), (List<List<Object>>) in.readGenericValue());
     }
 
     static void writeShowExec(PlanStreamOutput out, ShowExec showExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         writeAttributes(out, showExec.output());
         out.writeGenericValue(showExec.values());
     }
 
     static TopNExec readTopNExec(PlanStreamInput in) throws IOException {
         return new TopNExec(
-            in.readSource(),
+            Source.readFrom(in),
             in.readPhysicalPlanNode(),
             in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder)),
             in.readNamed(Expression.class),
@@ -782,7 +788,7 @@ static TopNExec readTopNExec(PlanStreamInput in) throws IOException {
     }
 
     static void writeTopNExec(PlanStreamOutput out, TopNExec topNExec) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(topNExec.child());
         out.writeCollection(topNExec.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder));
         out.writeExpression(topNExec.limit());
@@ -792,7 +798,7 @@ static void writeTopNExec(PlanStreamOutput out, TopNExec topNExec) throws IOExce
     // -- Logical plan nodes
     static Aggregate readAggregate(PlanStreamInput in) throws IOException {
         return new Aggregate(
-            in.readSource(),
+            Source.readFrom(in),
             in.readLogicalPlanNode(),
             in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)),
             readNamedExpressions(in)
@@ -800,18 +806,18 @@ static Aggregate readAggregate(PlanStreamInput in) throws IOException {
     }
 
     static void writeAggregate(PlanStreamOutput out, Aggregate aggregate) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(aggregate.child());
         out.writeCollection(aggregate.groupings(), writerFromPlanWriter(PlanStreamOutput::writeExpression));
         writeNamedExpressions(out, aggregate.aggregates());
     }
 
     static Dissect readDissect(PlanStreamInput in) throws IOException {
-        return new Dissect(in.readSource(), in.readLogicalPlanNode(), in.readExpression(), readDissectParser(in), readAttributes(in));
+        return new Dissect(Source.readFrom(in), in.readLogicalPlanNode(), in.readExpression(), readDissectParser(in), readAttributes(in));
     }
 
     static void writeDissect(PlanStreamOutput out, Dissect dissect) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(dissect.child());
         out.writeExpression(dissect.input());
         writeDissectParser(out, dissect.parser());
@@ -819,7 +825,7 @@ static void writeDissect(PlanStreamOutput out, Dissect dissect) throws IOExcepti
     }
 
     static EsRelation readEsRelation(PlanStreamInput in) throws IOException {
-        Source source = in.readSource();
+        Source source = Source.readFrom(in);
         EsIndex esIndex = readEsIndex(in);
         List<Attribute> attributes = readAttributes(in);
         if (supportingEsSourceOptions(in.getTransportVersion())) {
@@ -832,7 +838,7 @@ static EsRelation readEsRelation(PlanStreamInput in) throws IOException {
 
     static void writeEsRelation(PlanStreamOutput out, EsRelation relation) throws IOException {
         assert relation.children().size() == 0;
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         writeEsIndex(out, relation.index());
         writeAttributes(out, relation.output());
         if (supportingEsSourceOptions(out.getTransportVersion())) {
@@ -866,11 +872,11 @@ private static void writeEsSourceOptions(PlanStreamOutput out) throws IOExceptio
     }
 
     static Eval readEval(PlanStreamInput in) throws IOException {
-        return new Eval(in.readSource(), in.readLogicalPlanNode(), readAliases(in));
+        return new Eval(Source.readFrom(in), in.readLogicalPlanNode(), readAliases(in));
     }
 
     static void writeEval(PlanStreamOutput out, Eval eval) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(eval.child());
         writeAliases(out, eval.fields());
     }
@@ -880,7 +886,7 @@ static Enrich readEnrich(PlanStreamInput in) throws IOException {
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
             mode = in.readEnum(Enrich.Mode.class);
         }
-        final Source source = in.readSource();
+        final Source source = Source.readFrom(in);
         final LogicalPlan child = in.readLogicalPlanNode();
         final Expression policyName = in.readExpression();
         final NamedExpression matchField = in.readNamedExpression();
@@ -906,7 +912,7 @@ static void writeEnrich(PlanStreamOutput out, Enrich enrich) throws IOException
             out.writeEnum(enrich.mode());
         }
 
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(enrich.child());
         out.writeExpression(enrich.policyName());
         out.writeNamedExpression(enrich.matchField());
@@ -930,21 +936,21 @@ static void writeEnrich(PlanStreamOutput out, Enrich enrich) throws IOException
     }
 
     static EsqlProject readEsqlProject(PlanStreamInput in) throws IOException {
-        return new EsqlProject(in.readSource(), in.readLogicalPlanNode(), readNamedExpressions(in));
+        return new EsqlProject(Source.readFrom(in), in.readLogicalPlanNode(), readNamedExpressions(in));
     }
 
     static void writeEsqlProject(PlanStreamOutput out, EsqlProject project) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(project.child());
         writeNamedExpressions(out, project.projections());
     }
 
     static Filter readFilter(PlanStreamInput in) throws IOException {
-        return new Filter(in.readSource(), in.readLogicalPlanNode(), in.readExpression());
+        return new Filter(Source.readFrom(in), in.readLogicalPlanNode(), in.readExpression());
     }
 
     static void writeFilter(PlanStreamOutput out, Filter filter) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(filter.child());
         out.writeExpression(filter.condition());
     }
@@ -952,7 +958,7 @@ static void writeFilter(PlanStreamOutput out, Filter filter) throws IOException
     static Grok readGrok(PlanStreamInput in) throws IOException {
         Source source;
         return new Grok(
-            source = in.readSource(),
+            source = Source.readFrom(in),
             in.readLogicalPlanNode(),
             in.readExpression(),
             Grok.pattern(source, in.readString()),
@@ -961,7 +967,7 @@ static Grok readGrok(PlanStreamInput in) throws IOException {
     }
 
     static void writeGrok(PlanStreamOutput out, Grok grok) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(grok.child());
         out.writeExpression(grok.input());
         out.writeString(grok.parser().pattern());
@@ -969,21 +975,21 @@ static void writeGrok(PlanStreamOutput out, Grok grok) throws IOException {
     }
 
     static Limit readLimit(PlanStreamInput in) throws IOException {
-        return new Limit(in.readSource(), in.readNamed(Expression.class), in.readLogicalPlanNode());
+        return new Limit(Source.readFrom(in), in.readNamed(Expression.class), in.readLogicalPlanNode());
     }
 
     static void writeLimit(PlanStreamOutput out, Limit limit) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeExpression(limit.limit());
         out.writeLogicalPlanNode(limit.child());
     }
 
     static MvExpand readMvExpand(PlanStreamInput in) throws IOException {
-        return new MvExpand(in.readSource(), in.readLogicalPlanNode(), in.readNamedExpression(), in.readAttribute());
+        return new MvExpand(Source.readFrom(in), in.readLogicalPlanNode(), in.readNamedExpression(), in.readAttribute());
     }
 
     static void writeMvExpand(PlanStreamOutput out, MvExpand mvExpand) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(mvExpand.child());
         out.writeNamedExpression(mvExpand.target());
         out.writeAttribute(mvExpand.expanded());
@@ -991,31 +997,31 @@ static void writeMvExpand(PlanStreamOutput out, MvExpand mvExpand) throws IOExce
 
     static OrderBy readOrderBy(PlanStreamInput in) throws IOException {
         return new OrderBy(
-            in.readSource(),
+            Source.readFrom(in),
             in.readLogicalPlanNode(),
             in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder))
         );
     }
 
     static void writeOrderBy(PlanStreamOutput out, OrderBy order) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(order.child());
         out.writeCollection(order.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder));
     }
 
     static Project readProject(PlanStreamInput in) throws IOException {
-        return new Project(in.readSource(), in.readLogicalPlanNode(), readNamedExpressions(in));
+        return new Project(Source.readFrom(in), in.readLogicalPlanNode(), readNamedExpressions(in));
     }
 
     static void writeProject(PlanStreamOutput out, Project project) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(project.child());
         writeNamedExpressions(out, project.projections());
     }
 
     static TopN readTopN(PlanStreamInput in) throws IOException {
         return new TopN(
-            in.readSource(),
+            Source.readFrom(in),
             in.readLogicalPlanNode(),
             in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder)),
             in.readNamed(Expression.class)
@@ -1023,7 +1029,7 @@ static TopN readTopN(PlanStreamInput in) throws IOException {
     }
 
     static void writeTopN(PlanStreamOutput out, TopN topN) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(topN.child());
         out.writeCollection(topN.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder));
         out.writeExpression(topN.limit());
@@ -1059,7 +1065,7 @@ static void writeAliases(PlanStreamOutput out, List<Alias> aliases) throws IOExc
 
     static FieldAttribute readFieldAttribute(PlanStreamInput in) throws IOException {
         return new FieldAttribute(
-            in.readSource(),
+            Source.readFrom(in),
             in.readOptionalWithReader(PlanNamedTypes::readFieldAttribute),
             in.readString(),
             in.dataTypeFromTypeName(in.readString()),
@@ -1072,7 +1078,7 @@ static FieldAttribute readFieldAttribute(PlanStreamInput in) throws IOException
     }
 
     static void writeFieldAttribute(PlanStreamOutput out, FieldAttribute fileAttribute) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeOptionalWriteable(fileAttribute.parent() == null ? null : o -> writeFieldAttribute(out, fileAttribute.parent()));
         out.writeString(fileAttribute.name());
         out.writeString(fileAttribute.dataType().typeName());
@@ -1085,7 +1091,7 @@ static void writeFieldAttribute(PlanStreamOutput out, FieldAttribute fileAttribu
 
     static ReferenceAttribute readReferenceAttr(PlanStreamInput in) throws IOException {
         return new ReferenceAttribute(
-            in.readSource(),
+            Source.readFrom(in),
             in.readString(),
             in.dataTypeFromTypeName(in.readString()),
             in.readOptionalString(),
@@ -1096,7 +1102,7 @@ static ReferenceAttribute readReferenceAttr(PlanStreamInput in) throws IOExcepti
     }
 
     static void writeReferenceAttr(PlanStreamOutput out, ReferenceAttribute referenceAttribute) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeString(referenceAttribute.name());
         out.writeString(referenceAttribute.dataType().typeName());
         out.writeOptionalString(referenceAttribute.qualifier());
@@ -1107,7 +1113,7 @@ static void writeReferenceAttr(PlanStreamOutput out, ReferenceAttribute referenc
 
     static MetadataAttribute readMetadataAttr(PlanStreamInput in) throws IOException {
         return new MetadataAttribute(
-            in.readSource(),
+            Source.readFrom(in),
             in.readString(),
             in.dataTypeFromTypeName(in.readString()),
             in.readOptionalString(),
@@ -1119,7 +1125,7 @@ static MetadataAttribute readMetadataAttr(PlanStreamInput in) throws IOException
     }
 
     static void writeMetadataAttr(PlanStreamOutput out, MetadataAttribute metadataAttribute) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeString(metadataAttribute.name());
         out.writeString(metadataAttribute.dataType().typeName());
         out.writeOptionalString(metadataAttribute.qualifier());
@@ -1131,7 +1137,7 @@ static void writeMetadataAttr(PlanStreamOutput out, MetadataAttribute metadataAt
 
     static UnsupportedAttribute readUnsupportedAttr(PlanStreamInput in) throws IOException {
         return new UnsupportedAttribute(
-            in.readSource(),
+            Source.readFrom(in),
             in.readString(),
             readUnsupportedEsField(in),
             in.readOptionalString(),
@@ -1140,7 +1146,7 @@ static UnsupportedAttribute readUnsupportedAttr(PlanStreamInput in) throws IOExc
     }
 
     static void writeUnsupportedAttr(PlanStreamOutput out, UnsupportedAttribute unsupportedAttribute) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeString(unsupportedAttribute.name());
         writeUnsupportedEsField(out, unsupportedAttribute.field());
         out.writeOptionalString(unsupportedAttribute.hasCustomMessage() ? unsupportedAttribute.unresolvedMessage() : null);
@@ -1250,7 +1256,7 @@ static void writeUnsupportedEsField(PlanStreamOutput out, UnsupportedEsField uns
     // -- BinaryComparison
 
     static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) throws IOException {
-        var source = in.readSource();
+        var source = Source.readFrom(in);
         EsqlBinaryComparison.BinaryComparisonOperation operation = EsqlBinaryComparison.BinaryComparisonOperation.readFromStream(in);
         var left = in.readExpression();
         var right = in.readExpression();
@@ -1260,7 +1266,7 @@ static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) t
     }
 
     static void writeBinComparison(PlanStreamOutput out, EsqlBinaryComparison binaryComparison) throws IOException {
-        out.writeSource(binaryComparison.source());
+        binaryComparison.source().writeTo(out);
         binaryComparison.getFunctionType().writeTo(out);
         out.writeExpression(binaryComparison.left());
         out.writeExpression(binaryComparison.right());
@@ -1269,14 +1275,14 @@ static void writeBinComparison(PlanStreamOutput out, EsqlBinaryComparison binary
 
     // -- InsensitiveEquals
     static InsensitiveEquals readInsensitiveEquals(PlanStreamInput in, String name) throws IOException {
-        var source = in.readSource();
+        var source = Source.readFrom(in);
         var left = in.readExpression();
         var right = in.readExpression();
         return new InsensitiveEquals(source, left, right);
     }
 
     static void writeInsensitiveEquals(PlanStreamOutput out, InsensitiveEquals eq) throws IOException {
-        out.writeSource(eq.source());
+        eq.source().writeTo(out);
         out.writeExpression(eq.left());
         out.writeExpression(eq.right());
     }
@@ -1284,11 +1290,15 @@ static void writeInsensitiveEquals(PlanStreamOutput out, InsensitiveEquals eq) t
     // -- InComparison
 
     static In readInComparison(PlanStreamInput in) throws IOException {
-        return new In(in.readSource(), in.readExpression(), in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)));
+        return new In(
+            Source.readFrom(in),
+            in.readExpression(),
+            in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression))
+        );
     }
 
     static void writeInComparison(PlanStreamOutput out, In in) throws IOException {
-        out.writeSource(in.source());
+        in.source().writeTo(out);
         out.writeExpression(in.value());
         out.writeCollection(in.list(), writerFromPlanWriter(PlanStreamOutput::writeExpression));
     }
@@ -1296,21 +1306,21 @@ static void writeInComparison(PlanStreamOutput out, In in) throws IOException {
     // -- RegexMatch
 
     static WildcardLike readWildcardLike(PlanStreamInput in, String name) throws IOException {
-        return new WildcardLike(in.readSource(), in.readExpression(), new WildcardPattern(in.readString()));
+        return new WildcardLike(Source.readFrom(in), in.readExpression(), new WildcardPattern(in.readString()));
     }
 
     static void writeWildcardLike(PlanStreamOutput out, WildcardLike like) throws IOException {
-        out.writeSource(like.source());
+        like.source().writeTo(out);
         out.writeExpression(like.field());
         out.writeString(like.pattern().pattern());
     }
 
     static RLike readRLike(PlanStreamInput in, String name) throws IOException {
-        return new RLike(in.readSource(), in.readExpression(), new RLikePattern(in.readString()));
+        return new RLike(Source.readFrom(in), in.readExpression(), new RLikePattern(in.readString()));
     }
 
     static void writeRLike(PlanStreamOutput out, RLike like) throws IOException {
-        out.writeSource(like.source());
+        like.source().writeTo(out);
         out.writeExpression(like.field());
         out.writeString(like.pattern().asJavaRegex());
     }
@@ -1323,14 +1333,14 @@ static void writeRLike(PlanStreamOutput out, RLike like) throws IOException {
     );
 
     static BinaryLogic readBinaryLogic(PlanStreamInput in, String name) throws IOException {
-        var source = in.readSource();
+        var source = Source.readFrom(in);
         var left = in.readExpression();
         var right = in.readExpression();
         return BINARY_LOGIC_CTRS.get(name).apply(source, left, right);
     }
 
     static void writeBinaryLogic(PlanStreamOutput out, BinaryLogic binaryLogic) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeExpression(binaryLogic.left());
         out.writeExpression(binaryLogic.right());
     }
@@ -1385,11 +1395,11 @@ static UnaryScalarFunction readESQLUnaryScalar(PlanStreamInput in, String name)
         if (ctr == null) {
             throw new IOException("Constructor for ESQLUnaryScalar not found for name:" + name);
         }
-        return ctr.apply(in.readSource(), in.readExpression());
+        return ctr.apply(Source.readFrom(in), in.readExpression());
     }
 
     static void writeESQLUnaryScalar(PlanStreamOutput out, UnaryScalarFunction function) throws IOException {
-        out.writeSource(function.source());
+        function.source().writeTo(out);
         out.writeExpression(function.field());
     }
 
@@ -1404,11 +1414,11 @@ static ScalarFunction readNoArgScalar(PlanStreamInput in, String name) throws IO
         if (ctr == null) {
             throw new IOException("Constructor not found:" + name);
         }
-        return ctr.apply(in.readSource());
+        return ctr.apply(Source.readFrom(in));
     }
 
     static void writeNoArgScalar(PlanStreamOutput out, ScalarFunction function) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
     }
 
     static final Map<
@@ -1430,32 +1440,32 @@ static org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarF
         if (ctr == null) {
             throw new IOException("Constructor for QLUnaryScalar not found for name:" + name);
         }
-        return ctr.apply(in.readSource(), in.readExpression());
+        return ctr.apply(Source.readFrom(in), in.readExpression());
     }
 
     static void writeQLUnaryScalar(
         PlanStreamOutput out,
         org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction function
     ) throws IOException {
-        out.writeSource(function.source());
+        function.source().writeTo(out);
         out.writeExpression(function.field());
     }
 
     // -- ScalarFunction
 
     static Atan2 readAtan2(PlanStreamInput in) throws IOException {
-        return new Atan2(in.readSource(), in.readExpression(), in.readExpression());
+        return new Atan2(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
 
     static void writeAtan2(PlanStreamOutput out, Atan2 atan2) throws IOException {
-        out.writeSource(atan2.source());
+        atan2.source().writeTo(out);
         out.writeExpression(atan2.y());
         out.writeExpression(atan2.x());
     }
 
     static Bucket readBucket(PlanStreamInput in) throws IOException {
         return new Bucket(
-            in.readSource(),
+            Source.readFrom(in),
             in.readExpression(),
             in.readExpression(),
             in.readOptionalNamed(Expression.class),
@@ -1464,7 +1474,7 @@ static Bucket readBucket(PlanStreamInput in) throws IOException {
     }
 
     static void writeBucket(PlanStreamOutput out, Bucket bucket) throws IOException {
-        out.writeSource(bucket.source());
+        bucket.source().writeTo(out);
         out.writeExpression(bucket.field());
         out.writeExpression(bucket.buckets());
         out.writeOptionalExpression(bucket.from());
@@ -1481,11 +1491,15 @@ static void writeBucket(PlanStreamOutput out, Bucket bucket) throws IOException
 
     static ScalarFunction readVarag(PlanStreamInput in, String name) throws IOException {
         return VARARG_CTORS.get(name)
-            .apply(in.readSource(), in.readExpression(), in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)));
+            .apply(
+                Source.readFrom(in),
+                in.readExpression(),
+                in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression))
+            );
     }
 
     static void writeVararg(PlanStreamOutput out, ScalarFunction vararg) throws IOException {
-        out.writeSource(vararg.source());
+        vararg.source().writeTo(out);
         out.writeExpression(vararg.children().get(0));
         out.writeCollection(
             vararg.children().subList(1, vararg.children().size()),
@@ -1494,23 +1508,23 @@ static void writeVararg(PlanStreamOutput out, ScalarFunction vararg) throws IOEx
     }
 
     static CountDistinct readCountDistinct(PlanStreamInput in) throws IOException {
-        return new CountDistinct(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class));
+        return new CountDistinct(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class));
     }
 
     static void writeCountDistinct(PlanStreamOutput out, CountDistinct countDistinct) throws IOException {
         List<Expression> fields = countDistinct.children();
         assert fields.size() == 1 || fields.size() == 2;
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeExpression(fields.get(0));
         out.writeOptionalWriteable(fields.size() == 2 ? o -> out.writeExpression(fields.get(1)) : null);
     }
 
     static DateDiff readDateDiff(PlanStreamInput in) throws IOException {
-        return new DateDiff(in.readSource(), in.readExpression(), in.readExpression(), in.readExpression());
+        return new DateDiff(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readExpression());
     }
 
     static void writeDateDiff(PlanStreamOutput out, DateDiff function) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         List<Expression> fields = function.children();
         assert fields.size() == 3;
         out.writeExpression(fields.get(0));
@@ -1519,11 +1533,11 @@ static void writeDateDiff(PlanStreamOutput out, DateDiff function) throws IOExce
     }
 
     static DateExtract readDateExtract(PlanStreamInput in) throws IOException {
-        return new DateExtract(in.readSource(), in.readExpression(), in.readExpression(), in.configuration());
+        return new DateExtract(Source.readFrom(in), in.readExpression(), in.readExpression(), in.configuration());
     }
 
     static void writeDateExtract(PlanStreamOutput out, DateExtract function) throws IOException {
-        out.writeSource(function.source());
+        function.source().writeTo(out);
         List<Expression> fields = function.children();
         assert fields.size() == 2;
         out.writeExpression(fields.get(0));
@@ -1531,11 +1545,11 @@ static void writeDateExtract(PlanStreamOutput out, DateExtract function) throws
     }
 
     static DateFormat readDateFormat(PlanStreamInput in) throws IOException {
-        return new DateFormat(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class), in.configuration());
+        return new DateFormat(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class), in.configuration());
     }
 
     static void writeDateFormat(PlanStreamOutput out, DateFormat dateFormat) throws IOException {
-        out.writeSource(dateFormat.source());
+        dateFormat.source().writeTo(out);
         List<Expression> fields = dateFormat.children();
         assert fields.size() == 1 || fields.size() == 2;
         out.writeExpression(fields.get(0));
@@ -1543,11 +1557,11 @@ static void writeDateFormat(PlanStreamOutput out, DateFormat dateFormat) throws
     }
 
     static DateParse readDateTimeParse(PlanStreamInput in) throws IOException {
-        return new DateParse(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class));
+        return new DateParse(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class));
     }
 
     static void writeDateTimeParse(PlanStreamOutput out, DateParse function) throws IOException {
-        out.writeSource(function.source());
+        function.source().writeTo(out);
         List<Expression> fields = function.children();
         assert fields.size() == 1 || fields.size() == 2;
         out.writeExpression(fields.get(0));
@@ -1555,11 +1569,11 @@ static void writeDateTimeParse(PlanStreamOutput out, DateParse function) throws
     }
 
     static DateTrunc readDateTrunc(PlanStreamInput in) throws IOException {
-        return new DateTrunc(in.readSource(), in.readExpression(), in.readExpression());
+        return new DateTrunc(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
 
     static void writeDateTrunc(PlanStreamOutput out, DateTrunc dateTrunc) throws IOException {
-        out.writeSource(dateTrunc.source());
+        dateTrunc.source().writeTo(out);
         List<Expression> fields = dateTrunc.children();
         assert fields.size() == 2;
         out.writeExpression(fields.get(0));
@@ -1588,51 +1602,51 @@ static void writeSpatialRelatesFunction(PlanStreamOutput out, SpatialRelatesFunc
     }
 
     static Now readNow(PlanStreamInput in) throws IOException {
-        return new Now(in.readSource(), in.configuration());
+        return new Now(Source.readFrom(in), in.configuration());
     }
 
     static void writeNow(PlanStreamOutput out, Now function) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
     }
 
     static Round readRound(PlanStreamInput in) throws IOException {
-        return new Round(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class));
+        return new Round(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class));
     }
 
     static void writeRound(PlanStreamOutput out, Round round) throws IOException {
-        out.writeSource(round.source());
+        round.source().writeTo(out);
         out.writeExpression(round.field());
         out.writeOptionalExpression(round.decimals());
     }
 
     static Pow readPow(PlanStreamInput in) throws IOException {
-        return new Pow(in.readSource(), in.readExpression(), in.readExpression());
+        return new Pow(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
 
     static void writePow(PlanStreamOutput out, Pow pow) throws IOException {
-        out.writeSource(pow.source());
+        pow.source().writeTo(out);
         out.writeExpression(pow.base());
         out.writeExpression(pow.exponent());
     }
 
     static Percentile readPercentile(PlanStreamInput in) throws IOException {
-        return new Percentile(in.readSource(), in.readExpression(), in.readExpression());
+        return new Percentile(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
 
     static void writePercentile(PlanStreamOutput out, Percentile percentile) throws IOException {
         List<Expression> fields = percentile.children();
         assert fields.size() == 2 : "percentile() aggregation must have two arguments";
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeExpression(fields.get(0));
         out.writeExpression(fields.get(1));
     }
 
     static StartsWith readStartsWith(PlanStreamInput in) throws IOException {
-        return new StartsWith(in.readSource(), in.readExpression(), in.readExpression());
+        return new StartsWith(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
 
     static void writeStartsWith(PlanStreamOutput out, StartsWith startsWith) throws IOException {
-        out.writeSource(startsWith.source());
+        startsWith.source().writeTo(out);
         List<Expression> fields = startsWith.children();
         assert fields.size() == 2;
         out.writeExpression(fields.get(0));
@@ -1640,23 +1654,23 @@ static void writeStartsWith(PlanStreamOutput out, StartsWith startsWith) throws
     }
 
     static EndsWith readEndsWith(PlanStreamInput in) throws IOException {
-        return new EndsWith(in.readSource(), in.readExpression(), in.readExpression());
+        return new EndsWith(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
 
     static void writeEndsWith(PlanStreamOutput out, EndsWith endsWith) throws IOException {
         List<Expression> fields = endsWith.children();
         assert fields.size() == 2;
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeExpression(fields.get(0));
         out.writeExpression(fields.get(1));
     }
 
     static Substring readSubstring(PlanStreamInput in) throws IOException {
-        return new Substring(in.readSource(), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class));
+        return new Substring(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class));
     }
 
     static void writeSubstring(PlanStreamOutput out, Substring substring) throws IOException {
-        out.writeSource(substring.source());
+        substring.source().writeTo(out);
         List<Expression> fields = substring.children();
         assert fields.size() == 2 || fields.size() == 3;
         out.writeExpression(fields.get(0));
@@ -1665,11 +1679,11 @@ static void writeSubstring(PlanStreamOutput out, Substring substring) throws IOE
     }
 
     static Locate readLocate(PlanStreamInput in) throws IOException {
-        return new Locate(in.readSource(), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class));
+        return new Locate(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class));
     }
 
     static void writeLocate(PlanStreamOutput out, Locate locate) throws IOException {
-        out.writeSource(locate.source());
+        locate.source().writeTo(out);
         List<Expression> fields = locate.children();
         assert fields.size() == 2 || fields.size() == 3;
         out.writeExpression(fields.get(0));
@@ -1706,11 +1720,11 @@ static void writeToUpper(PlanStreamOutput out, ToUpper toUpper) throws IOExcepti
     }
 
     static Left readLeft(PlanStreamInput in) throws IOException {
-        return new Left(in.readSource(), in.readExpression(), in.readExpression());
+        return new Left(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
 
     static void writeLeft(PlanStreamOutput out, Left left) throws IOException {
-        out.writeSource(left.source());
+        left.source().writeTo(out);
         List<Expression> fields = left.children();
         assert fields.size() == 2;
         out.writeExpression(fields.get(0));
@@ -1718,11 +1732,11 @@ static void writeLeft(PlanStreamOutput out, Left left) throws IOException {
     }
 
     static Right readRight(PlanStreamInput in) throws IOException {
-        return new Right(in.readSource(), in.readExpression(), in.readExpression());
+        return new Right(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
 
     static void writeRight(PlanStreamOutput out, Right right) throws IOException {
-        out.writeSource(right.source());
+        right.source().writeTo(out);
         List<Expression> fields = right.children();
         assert fields.size() == 2;
         out.writeExpression(fields.get(0));
@@ -1730,25 +1744,25 @@ static void writeRight(PlanStreamOutput out, Right right) throws IOException {
     }
 
     static Split readSplit(PlanStreamInput in) throws IOException {
-        return new Split(in.readSource(), in.readExpression(), in.readExpression());
+        return new Split(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
 
     static void writeSplit(PlanStreamOutput out, Split split) throws IOException {
-        out.writeSource(split.source());
+        split.source().writeTo(out);
         out.writeExpression(split.left());
         out.writeExpression(split.right());
     }
 
     static CIDRMatch readCIDRMatch(PlanStreamInput in) throws IOException {
         return new CIDRMatch(
-            in.readSource(),
+            Source.readFrom(in),
             in.readExpression(),
             in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression))
         );
     }
 
     static void writeCIDRMatch(PlanStreamOutput out, CIDRMatch cidrMatch) throws IOException {
-        out.writeSource(cidrMatch.source());
+        cidrMatch.source().writeTo(out);
         List<Expression> children = cidrMatch.children();
         assert children.size() > 1;
         out.writeExpression(children.get(0));
@@ -1766,14 +1780,14 @@ static void writeCIDRMatch(PlanStreamOutput out, CIDRMatch cidrMatch) throws IOE
     );
 
     static ArithmeticOperation readArithmeticOperation(PlanStreamInput in, String name) throws IOException {
-        var source = in.readSource();
+        var source = Source.readFrom(in);
         var left = in.readExpression();
         var right = in.readExpression();
         return ARITHMETIC_CTRS.get(name).apply(source, left, right);
     }
 
     static void writeArithmeticOperation(PlanStreamOutput out, ArithmeticOperation arithmeticOperation) throws IOException {
-        out.writeSource(arithmeticOperation.source());
+        arithmeticOperation.source().writeTo(out);
         out.writeExpression(arithmeticOperation.left());
         out.writeExpression(arithmeticOperation.right());
     }
@@ -1792,11 +1806,11 @@ static void writeArithmeticOperation(PlanStreamOutput out, ArithmeticOperation a
     );
 
     static AggregateFunction readAggFunction(PlanStreamInput in, String name) throws IOException {
-        return AGG_CTRS.get(name).apply(in.readSource(), in.readExpression());
+        return AGG_CTRS.get(name).apply(Source.readFrom(in), in.readExpression());
     }
 
     static void writeAggFunction(PlanStreamOutput out, AggregateFunction aggregateFunction) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeExpression(aggregateFunction.field());
     }
 
@@ -1814,20 +1828,20 @@ static void writeAggFunction(PlanStreamOutput out, AggregateFunction aggregateFu
     );
 
     static AbstractMultivalueFunction readMvFunction(PlanStreamInput in, String name) throws IOException {
-        return MV_CTRS.get(name).apply(in.readSource(), in.readExpression());
+        return MV_CTRS.get(name).apply(Source.readFrom(in), in.readExpression());
     }
 
     static void writeMvFunction(PlanStreamOutput out, AbstractMultivalueFunction fn) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeExpression(fn.field());
     }
 
     static MvConcat readMvConcat(PlanStreamInput in) throws IOException {
-        return new MvConcat(in.readSource(), in.readExpression(), in.readExpression());
+        return new MvConcat(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
 
     static void writeMvConcat(PlanStreamOutput out, MvConcat fn) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeExpression(fn.left());
         out.writeExpression(fn.right());
     }
@@ -1836,7 +1850,7 @@ static void writeMvConcat(PlanStreamOutput out, MvConcat fn) throws IOException
 
     static Alias readAlias(PlanStreamInput in) throws IOException {
         return new Alias(
-            in.readSource(),
+            Source.readFrom(in),
             in.readString(),
             in.readOptionalString(),
             in.readNamed(Expression.class),
@@ -1846,7 +1860,7 @@ static Alias readAlias(PlanStreamInput in) throws IOException {
     }
 
     static void writeAlias(PlanStreamOutput out, Alias alias) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeString(alias.name());
         out.writeOptionalString(alias.qualifier());
         out.writeExpression(alias.child());
@@ -1857,14 +1871,14 @@ static void writeAlias(PlanStreamOutput out, Alias alias) throws IOException {
     // -- Expressions (other)
 
     static Literal readLiteral(PlanStreamInput in) throws IOException {
-        Source source = in.readSource();
+        Source source = Source.readFrom(in);
         Object value = in.readGenericValue();
         DataType dataType = in.dataTypeFromTypeName(in.readString());
         return new Literal(source, mapToLiteralValue(in, dataType, value), dataType);
     }
 
     static void writeLiteral(PlanStreamOutput out, Literal literal) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeGenericValue(mapFromLiteralValue(out, literal.dataType(), literal.value()));
         out.writeString(literal.dataType().typeName());
     }
@@ -1917,7 +1931,7 @@ private static long wkbAsLong(DataType dataType, BytesRef wkb) {
 
     static Order readOrder(PlanStreamInput in) throws IOException {
         return new org.elasticsearch.xpack.esql.expression.Order(
-            in.readSource(),
+            Source.readFrom(in),
             in.readNamed(Expression.class),
             in.readEnum(Order.OrderDirection.class),
             in.readEnum(Order.NullsPosition.class)
@@ -1925,7 +1939,7 @@ static Order readOrder(PlanStreamInput in) throws IOException {
     }
 
     static void writeOrder(PlanStreamOutput out, Order order) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         out.writeExpression(order.child());
         out.writeEnum(order.direction());
         out.writeEnum(order.nullsPosition());
@@ -1974,11 +1988,11 @@ static void writeDissectParser(PlanStreamOutput out, Parser dissectParser) throw
     }
 
     static Log readLog(PlanStreamInput in) throws IOException {
-        return new Log(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class));
+        return new Log(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class));
     }
 
     static void writeLog(PlanStreamOutput out, Log log) throws IOException {
-        out.writeSource(log.source());
+        log.source().writeTo(out);
         List<Expression> fields = log.children();
         assert fields.size() == 1 || fields.size() == 2;
         out.writeExpression(fields.get(0));
@@ -1986,11 +2000,11 @@ static void writeLog(PlanStreamOutput out, Log log) throws IOException {
     }
 
     static MvSort readMvSort(PlanStreamInput in) throws IOException {
-        return new MvSort(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class));
+        return new MvSort(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class));
     }
 
     static void writeMvSort(PlanStreamOutput out, MvSort mvSort) throws IOException {
-        out.writeSource(mvSort.source());
+        mvSort.source().writeTo(out);
         List<Expression> fields = mvSort.children();
         assert fields.size() == 1 || fields.size() == 2;
         out.writeExpression(fields.get(0));
@@ -1998,11 +2012,11 @@ static void writeMvSort(PlanStreamOutput out, MvSort mvSort) throws IOException
     }
 
     static MvSlice readMvSlice(PlanStreamInput in) throws IOException {
-        return new MvSlice(in.readSource(), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class));
+        return new MvSlice(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class));
     }
 
     static void writeMvSlice(PlanStreamOutput out, MvSlice fn) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         List<Expression> fields = fn.children();
         assert fields.size() == 2 || fields.size() == 3;
         out.writeExpression(fields.get(0));
@@ -2011,11 +2025,11 @@ static void writeMvSlice(PlanStreamOutput out, MvSlice fn) throws IOException {
     }
 
     static MvZip readMvZip(PlanStreamInput in) throws IOException {
-        return new MvZip(in.readSource(), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class));
+        return new MvZip(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class));
     }
 
     static void writeMvZip(PlanStreamOutput out, MvZip fn) throws IOException {
-        out.writeNoSource();
+        Source.EMPTY.writeTo(out);
         List<Expression> fields = fn.children();
         assert fields.size() == 2 || fields.size() == 3;
         out.writeExpression(fields.get(0));
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
index 3faf85b989c09..c8dc24e4c2208 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanNamedReader;
@@ -48,13 +47,13 @@
 import java.util.function.LongFunction;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.xpack.esql.core.util.SourceUtils.readSourceWithText;
-
 /**
  * A customized stream input used to deserialize ESQL physical plan fragments. Complements stream
  * input with methods that read plan nodes, Attributes, Expressions, etc.
  */
-public final class PlanStreamInput extends NamedWriteableAwareStreamInput {
+public final class PlanStreamInput extends NamedWriteableAwareStreamInput
+    implements
+        org.elasticsearch.xpack.esql.core.util.PlanStreamInput {
 
     /**
      * A Mapper of stream named id, represented as a primitive long value, to NameId instance.
@@ -123,11 +122,6 @@ public PhysicalPlan readOptionalPhysicalPlanNode() throws IOException {
         return readOptionalNamed(PhysicalPlan.class);
     }
 
-    public Source readSource() throws IOException {
-        boolean hasSource = readBoolean();
-        return hasSource ? readSourceWithText(this, configuration.query()) : Source.EMPTY;
-    }
-
     public Expression readExpression() throws IOException {
         return readNamed(Expression.class);
     }
@@ -268,6 +262,11 @@ public Block[] readCachedBlockArray() throws IOException {
         }
     }
 
+    @Override
+    public String sourceText() {
+        return configuration.query();
+    }
+
     static void throwOnNullOptionalRead(Class<?> type) throws IOException {
         final IOException e = new IOException("read optional named returned null which is not allowed, type:" + type);
         assert false : e;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
index 6cf64de339383..351918699aac4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
@@ -33,8 +32,6 @@
 import java.util.Map;
 import java.util.function.Function;
 
-import static org.elasticsearch.xpack.esql.core.util.SourceUtils.writeSourceNoText;
-
 /**
  * A customized stream output used to serialize ESQL physical plan fragments. Complements stream
  * output with methods that write plan nodes, Attributes, Expressions, etc.
@@ -98,15 +95,6 @@ public void writeOptionalPhysicalPlanNode(PhysicalPlan physicalPlan) throws IOEx
         }
     }
 
-    public void writeSource(Source source) throws IOException {
-        writeBoolean(true);
-        writeSourceNoText(this, source);
-    }
-
-    public void writeNoSource() throws IOException {
-        writeBoolean(false);
-    }
-
     public void writeExpression(Expression expression) throws IOException {
         writeNamed(Expression.class, expression);
     }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/SourceUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/SourceUtils.java
deleted file mode 100644
index afba73373df92..0000000000000
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/SourceUtils.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.ql.util;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
-import org.elasticsearch.xpack.ql.tree.Location;
-import org.elasticsearch.xpack.ql.tree.Source;
-
-import java.io.IOException;
-
-public final class SourceUtils {
-
-    private SourceUtils() {}
-
-    public static void writeSource(StreamOutput out, Source source) throws IOException {
-        writeSource(out, source, true);
-    }
-
-    public static void writeSourceNoText(StreamOutput out, Source source) throws IOException {
-        writeSource(out, source, false);
-    }
-
-    public static Source readSource(StreamInput in) throws IOException {
-        return readSource(in, null);
-    }
-
-    public static Source readSourceWithText(StreamInput in, String queryText) throws IOException {
-        return readSource(in, queryText);
-    }
-
-    private static void writeSource(StreamOutput out, Source source, boolean writeText) throws IOException {
-        out.writeInt(source.source().getLineNumber());
-        out.writeInt(source.source().getColumnNumber());
-        if (writeText) {
-            out.writeString(source.text());
-        } else {
-            out.writeInt(source.text().length());
-        }
-    }
-
-    private static Source readSource(StreamInput in, @Nullable String queryText) throws IOException {
-        int line = in.readInt();
-        int column = in.readInt();
-        int charPositionInLine = column - 1;
-
-        String text;
-        if (queryText == null) {
-            text = in.readString();
-        } else {
-            int length = in.readInt();
-            text = sourceText(queryText, line, column, length);
-        }
-        return new Source(new Location(line, charPositionInLine), text);
-    }
-
-    private static String sourceText(String query, int line, int column, int length) {
-        if (line <= 0 || column <= 0 || query.isEmpty()) {
-            return StringUtils.EMPTY;
-        }
-        int offset = textOffset(query, line, column);
-        if (offset + length > query.length()) {
-            throw new QlIllegalArgumentException(
-                "location [@" + line + ":" + column + "] and length [" + length + "] overrun query size [" + query.length() + "]"
-            );
-        }
-        return query.substring(offset, offset + length);
-    }
-
-    private static int textOffset(String query, int line, int column) {
-        int offset = 0;
-        if (line > 1) {
-            String[] lines = query.split("\n");
-            if (line > lines.length) {
-                throw new QlIllegalArgumentException(
-                    "line location [" + line + "] higher than max [" + lines.length + "] in query [" + query + "]"
-                );
-            }
-            for (int i = 0; i < line - 1; i++) {
-                offset += lines[i].length() + 1; // +1 accounts for the removed \n
-            }
-        }
-        offset += column - 1; // -1 since column is 1-based indexed
-        return offset;
-    }
-}

From da9282e3f571e5af834953db679763bde8d1530d Mon Sep 17 00:00:00 2001
From: Henning Andersen <33268011+henningandersen@users.noreply.github.com>
Date: Wed, 29 May 2024 16:23:22 +0200
Subject: [PATCH 034/208] Longer timeout for local node startup during CI
 (#109141)

Sometimes CI is slow enough that the 2 min timeout fails. Extending it.

Seen in CI failure from #109139
---
 .../test/cluster/local/AbstractLocalClusterFactory.java         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java
index 9bd3403060b2a..0101c76b21f76 100644
--- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java
@@ -64,7 +64,7 @@ public abstract class AbstractLocalClusterFactory<S extends LocalClusterSpec, H
     implements
         LocalClusterFactory<S, H> {
     private static final Logger LOGGER = LogManager.getLogger(AbstractLocalClusterFactory.class);
-    private static final Duration NODE_UP_TIMEOUT = Duration.ofMinutes(2);
+    private static final Duration NODE_UP_TIMEOUT = Duration.ofMinutes(3);
     private static final Map<Pair<Version, DistributionType>, DistributionDescriptor> TEST_DISTRIBUTIONS = new ConcurrentHashMap<>();
     private static final String TESTS_CLUSTER_MODULES_PATH_SYSPROP = "tests.cluster.modules.path";
     private static final String TESTS_CLUSTER_PLUGINS_PATH_SYSPROP = "tests.cluster.plugins.path";

From f16f71e2a2bd025cb2bc63cad0670deb91017818 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= <ivancea96@outlook.com>
Date: Wed, 29 May 2024 16:23:45 +0200
Subject: [PATCH 035/208] ESQL: Add ip_prefix function (#109070)

Added ESQL function to get the prefix of an IP. It works now with both
IPv4 and IPv6. For users planning to use it with mixed IPs, we may need
to add a function like "is_ipv4()" first.

**About the skipped test:** There's currently a "bug" in the
evaluators//functions that return null. Evaluators can't handle them.
We'll work on support for that in another PR. It affects other
functions, like `substring()`. In this function, however, it only
affects in "wrong" cases (Like an invalid prefix), so it has no impact.

Fixes https://github.com/elastic/elasticsearch/issues/99064
---
 .gitattributes                                |   5 +
 docs/changelog/109070.yaml                    |   6 +
 .../functions/description/ip_prefix.asciidoc  |   5 +
 .../functions/examples/ip_prefix.asciidoc     |  13 ++
 .../kibana/definition/ip_prefix.json          |  35 ++++
 .../esql/functions/kibana/docs/ip_prefix.md   |  11 +
 .../esql/functions/layout/ip_prefix.asciidoc  |  15 ++
 .../functions/parameters/ip_prefix.asciidoc   |  12 ++
 .../esql/functions/signature/ip_prefix.svg    |   1 +
 .../esql/functions/types/ip_prefix.asciidoc   |   9 +
 .../xpack/esql/CsvTestUtils.java              |   2 +-
 .../src/main/resources/ip.csv-spec            | 105 ++++++++++
 .../src/main/resources/meta.csv-spec          |   6 +-
 .../function/scalar/ip/IpPrefixEvaluator.java | 183 +++++++++++++++++
 .../scalar/ip/IpPrefixOnlyV4Evaluator.java    | 148 ++++++++++++++
 .../xpack/esql/action/EsqlCapabilities.java   |   6 +
 .../function/EsqlFunctionRegistry.java        |   2 +
 .../function/scalar/ip/IpPrefix.java          | 191 ++++++++++++++++++
 .../xpack/esql/io/stream/PlanNamedTypes.java  |   2 +
 .../function/scalar/ip/IpPrefixTests.java     | 116 +++++++++++
 20 files changed, 871 insertions(+), 2 deletions(-)
 create mode 100644 docs/changelog/109070.yaml
 create mode 100644 docs/reference/esql/functions/description/ip_prefix.asciidoc
 create mode 100644 docs/reference/esql/functions/examples/ip_prefix.asciidoc
 create mode 100644 docs/reference/esql/functions/kibana/definition/ip_prefix.json
 create mode 100644 docs/reference/esql/functions/kibana/docs/ip_prefix.md
 create mode 100644 docs/reference/esql/functions/layout/ip_prefix.asciidoc
 create mode 100644 docs/reference/esql/functions/parameters/ip_prefix.asciidoc
 create mode 100644 docs/reference/esql/functions/signature/ip_prefix.svg
 create mode 100644 docs/reference/esql/functions/types/ip_prefix.asciidoc
 create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java
 create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java

diff --git a/.gitattributes b/.gitattributes
index ebe1a34db3479..6a8de5462ec3f 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -8,3 +8,8 @@ x-pack/plugin/esql/src/main/antlr/*.tokens linguist-generated=true
 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*.interp linguist-generated=true
 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer*.java linguist-generated=true
 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser*.java linguist-generated=true
+x-pack/plugin/esql/src/main/generated/** linguist-generated=true
+
+# ESQL functions docs are autogenerated. More information at `docs/reference/esql/functions/README.md`
+docs/reference/esql/functions/*/** linguist-generated=true
+
diff --git a/docs/changelog/109070.yaml b/docs/changelog/109070.yaml
new file mode 100644
index 0000000000000..8dbc0ec1c6cf2
--- /dev/null
+++ b/docs/changelog/109070.yaml
@@ -0,0 +1,6 @@
+pr: 109070
+summary: "ESQL: Add `ip_prefix` function"
+area: ES|QL
+type: feature
+issues:
+ - 99064
diff --git a/docs/reference/esql/functions/description/ip_prefix.asciidoc b/docs/reference/esql/functions/description/ip_prefix.asciidoc
new file mode 100644
index 0000000000000..4b7a88486dea2
--- /dev/null
+++ b/docs/reference/esql/functions/description/ip_prefix.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Truncates an IP to a given prefix length.
diff --git a/docs/reference/esql/functions/examples/ip_prefix.asciidoc b/docs/reference/esql/functions/examples/ip_prefix.asciidoc
new file mode 100644
index 0000000000000..19f0ed266afb5
--- /dev/null
+++ b/docs/reference/esql/functions/examples/ip_prefix.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/ip.csv-spec[tag=ipPrefix]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/ip.csv-spec[tag=ipPrefix-result]
+|===
+
diff --git a/docs/reference/esql/functions/kibana/definition/ip_prefix.json b/docs/reference/esql/functions/kibana/definition/ip_prefix.json
new file mode 100644
index 0000000000000..00c3cf75a949e
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/ip_prefix.json
@@ -0,0 +1,35 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "ip_prefix",
+  "description" : "Truncates an IP to a given prefix length.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "ip",
+          "type" : "ip",
+          "optional" : false,
+          "description" : "IP address of type `ip` (both IPv4 and IPv6 are supported)."
+        },
+        {
+          "name" : "prefixLengthV4",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Prefix length for IPv4 addresses."
+        },
+        {
+          "name" : "prefixLengthV6",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Prefix length for IPv6 addresses."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "ip"
+    }
+  ],
+  "examples" : [
+    "row ip4 = to_ip(\"1.2.3.4\"), ip6 = to_ip(\"fe80::cae2:65ff:fece:feb9\")\n| eval ip4_prefix = ip_prefix(ip4, 24, 0), ip6_prefix = ip_prefix(ip6, 0, 112);"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/docs/ip_prefix.md b/docs/reference/esql/functions/kibana/docs/ip_prefix.md
new file mode 100644
index 0000000000000..5c0009528bb68
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/ip_prefix.md
@@ -0,0 +1,11 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### IP_PREFIX
+Truncates an IP to a given prefix length.
+
+```
+row ip4 = to_ip("1.2.3.4"), ip6 = to_ip("fe80::cae2:65ff:fece:feb9")
+| eval ip4_prefix = ip_prefix(ip4, 24, 0), ip6_prefix = ip_prefix(ip6, 0, 112);
+```
diff --git a/docs/reference/esql/functions/layout/ip_prefix.asciidoc b/docs/reference/esql/functions/layout/ip_prefix.asciidoc
new file mode 100644
index 0000000000000..ca51c871daf7f
--- /dev/null
+++ b/docs/reference/esql/functions/layout/ip_prefix.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-ip_prefix]]
+=== `IP_PREFIX`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/ip_prefix.svg[Embedded,opts=inline]
+
+include::../parameters/ip_prefix.asciidoc[]
+include::../description/ip_prefix.asciidoc[]
+include::../types/ip_prefix.asciidoc[]
+include::../examples/ip_prefix.asciidoc[]
diff --git a/docs/reference/esql/functions/parameters/ip_prefix.asciidoc b/docs/reference/esql/functions/parameters/ip_prefix.asciidoc
new file mode 100644
index 0000000000000..945601c2476e6
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/ip_prefix.asciidoc
@@ -0,0 +1,12 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`ip`::
+IP address of type `ip` (both IPv4 and IPv6 are supported).
+
+`prefixLengthV4`::
+Prefix length for IPv4 addresses.
+
+`prefixLengthV6`::
+Prefix length for IPv6 addresses.
diff --git a/docs/reference/esql/functions/signature/ip_prefix.svg b/docs/reference/esql/functions/signature/ip_prefix.svg
new file mode 100644
index 0000000000000..4699c23357460
--- /dev/null
+++ b/docs/reference/esql/functions/signature/ip_prefix.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="756" height="46" viewbox="0 0 756 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m128 0h10m32 0h10m44 0h10m32 0h10m188 0h10m32 0h10m188 0h10m32 0h5"/><rect class="s" x="5" y="5" width="128" height="36"/><text class="k" x="15" y="31">IP_PREFIX</text><rect class="s" x="143" y="5" width="32" height="36" rx="7"/><text class="syn" x="153" y="31">(</text><rect class="s" x="185" y="5" width="44" height="36" rx="7"/><text class="k" x="195" y="31">ip</text><rect class="s" x="239" y="5" width="32" height="36" rx="7"/><text class="syn" x="249" y="31">,</text><rect class="s" x="281" y="5" width="188" height="36" rx="7"/><text class="k" x="291" y="31">prefixLengthV4</text><rect class="s" x="479" y="5" width="32" height="36" rx="7"/><text class="syn" x="489" y="31">,</text><rect class="s" x="521" y="5" width="188" height="36" rx="7"/><text class="k" x="531" y="31">prefixLengthV6</text><rect class="s" x="719" y="5" width="32" height="36" rx="7"/><text class="syn" x="729" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/types/ip_prefix.asciidoc b/docs/reference/esql/functions/types/ip_prefix.asciidoc
new file mode 100644
index 0000000000000..786d99d45d327
--- /dev/null
+++ b/docs/reference/esql/functions/types/ip_prefix.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+ip | prefixLengthV4 | prefixLengthV6 | result
+ip | integer | integer | ip
+|===
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
index 1927cfd03ac06..7d1c168bd203f 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
@@ -101,7 +101,7 @@ public static Tuple<Version, Version> skipVersionRange(String testName) {
         Map<String, String> pairs = extractInstructions(testName);
         String versionRange = pairs.get("skip");
         if (versionRange != null) {
-            String[] skipVersions = versionRange.split("-");
+            String[] skipVersions = versionRange.split("-", Integer.MAX_VALUE);
             if (skipVersions.length != 2) {
                 throw new IllegalArgumentException("malformed version range : " + versionRange);
             }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
index ae683acbb2c3a..0de64d3e2d9d4 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
@@ -485,3 +485,108 @@ beta         | 127.0.0.1
 beta         | 127.0.0.1
 beta         | 127.0.0.1
 ;
+
+ipPrefix
+required_capability: fn_ip_prefix
+//tag::ipPrefix[]
+row ip4 = to_ip("1.2.3.4"), ip6 = to_ip("fe80::cae2:65ff:fece:feb9")
+| eval ip4_prefix = ip_prefix(ip4, 24, 0), ip6_prefix = ip_prefix(ip6, 0, 112);
+//end::ipPrefix[]
+
+//tag::ipPrefix-result[]
+ip4:ip  | ip6:ip                    | ip4_prefix:ip | ip6_prefix:ip
+1.2.3.4 | fe80::cae2:65ff:fece:feb9 | 1.2.3.0       | fe80::cae2:65ff:fece:0000
+//end::ipPrefix-result[]
+;
+
+ipPrefixCompleteIp
+required_capability: fn_ip_prefix
+row ip4 = to_ip("1.2.3.4"), ip6 = to_ip("fe80::cae2:65ff:fece:feb9")
+| eval ip4_prefix = ip_prefix(ip4, 32, 0), ip6_prefix = ip_prefix(ip6, 0, 128);
+
+ip4:ip  | ip6:ip                    | ip4_prefix:ip | ip6_prefix:ip
+1.2.3.4 | fe80::cae2:65ff:fece:feb9 | 1.2.3.4       | fe80::cae2:65ff:fece:feb9 
+;
+
+ipPrefixZeroBits
+required_capability: fn_ip_prefix
+row ip4 = to_ip("1.2.3.4"), ip6 = to_ip("fe80::cae2:65ff:fece:feb9")
+| eval ip4_prefix = ip_prefix(ip4, 0, 128), ip6_prefix = ip_prefix(ip6, 32, 0);
+
+ip4:ip  | ip6:ip                    | ip4_prefix:ip | ip6_prefix:ip
+1.2.3.4 | fe80::cae2:65ff:fece:feb9 | 0.0.0.0       | ::0
+;
+
+ipPrefixWithBits
+required_capability: fn_ip_prefix
+row ip4 = to_ip("1.2.3.255"), ip6 = to_ip("fe80::cae2:65ff:fece:feff")
+| eval ip4_prefix = ip_prefix(ip4, 25, 0), ip6_prefix = ip_prefix(ip6, 0, 121);
+
+ip4:ip    | ip6:ip                    | ip4_prefix:ip | ip6_prefix:ip
+1.2.3.255 | fe80::cae2:65ff:fece:feff | 1.2.3.128     | fe80::cae2:65ff:fece:fe80
+;
+
+ipPrefixLengthFromColumn
+required_capability: fn_ip_prefix
+from hosts
+| where host == "alpha"
+| sort card
+| eval prefix = ip_prefix(ip0, 24, 128)
+| keep card, ip0, prefix;
+
+card:keyword | ip0:ip    | prefix:ip
+eth0         | 127.0.0.1 | 127.0.0.0
+eth1         | ::1       | ::0
+;
+
+ipPrefixLengthFromExpression
+required_capability: fn_ip_prefix
+row ip4 = to_ip("1.2.3.4"), ip6 = to_ip("fe80::cae2:65ff:fece:feb9"), bits_per_byte = 8
+| eval ip4_length = 3 * bits_per_byte, ip4_prefix = ip_prefix(ip4, ip4_length, 0), ip6_prefix = ip_prefix(ip6, 0, 12 * 10);
+
+ip4:ip  | ip6:ip                    | bits_per_byte:integer | ip4_length:integer | ip4_prefix:ip | ip6_prefix:ip
+1.2.3.4 | fe80::cae2:65ff:fece:feb9 | 8                     | 24                 | 1.2.3.0       | fe80::cae2:65ff:fece:fe00
+;
+
+ipPrefixAsGroup
+required_capability: fn_ip_prefix
+from hosts
+| stats count(*) by ip_prefix(ip1, 24, 120)
+| sort `ip_prefix(ip1, 24, 120)`;
+warning:Line 2:21: evaluation of [ip_prefix(ip1, 24, 120)] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:21: java.lang.IllegalArgumentException: single-value function encountered multi-value
+
+count(*):long | ip_prefix(ip1, 24, 120):ip
+2             | ::0
+3             | 127.0.0.0
+1             | 128.0.0.0
+1             | fe80::cae2:65ff:fece:fe00
+1             | fe81::cae2:65ff:fece:fe00
+2             | null
+;
+
+ipPrefixWithWrongLengths
+required_capability: fn_ip_prefix
+row ip4 = to_ip("1.2.3.4")
+| eval a = ip_prefix(ip4, -1, 128), b = ip_prefix(ip4, 32, -1), c = ip_prefix(ip4, 33, 0), d = ip_prefix(ip4, 32, 129);
+warning:Line 2:12: evaluation of [ip_prefix(ip4, -1, 128)] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:12: java.lang.IllegalArgumentException: Prefix length v4 must be in range [0, 32], found -1
+warning:Line 2:41: evaluation of [ip_prefix(ip4, 32, -1)] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:41: java.lang.IllegalArgumentException: Prefix length v6 must be in range [0, 128], found -1
+warning:Line 2:69: evaluation of [ip_prefix(ip4, 33, 0)] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:69: java.lang.IllegalArgumentException: Prefix length v4 must be in range [0, 32], found 33
+warning:Line 2:96: evaluation of [ip_prefix(ip4, 32, 129)] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 2:96: java.lang.IllegalArgumentException: Prefix length v6 must be in range [0, 128], found 129
+
+ip4:ip  | a:ip | b:ip | c:ip | d:ip
+1.2.3.4 | null | null | null | null
+;
+
+ipPrefixWithNullArguments
+required_capability: fn_ip_prefix
+row ip4 = to_ip("1.2.3.4")
+| eval a = ip_prefix(null, 32, 128), b = ip_prefix(ip4, null, 128), c = ip_prefix(ip4, 32, null);
+
+ip4:ip  | a:ip | b:ip | c:ip
+1.2.3.4 | null | null | null
+;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
index f68dd15e9c516..eff4cb05bd8c0 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
@@ -30,6 +30,7 @@ double e()
 "double|integer|long|unsigned_long floor(number:double|integer|long|unsigned_long)"
 "keyword from_base64(string:keyword|text)"
 "boolean|double|integer|ip|keyword|long|text|version greatest(first:boolean|double|integer|ip|keyword|long|text|version, ?rest...:boolean|double|integer|ip|keyword|long|text|version)"
+"ip ip_prefix(ip:ip, prefixLengthV4:integer, prefixLengthV6:integer)"
 "boolean|double|integer|ip|keyword|long|text|version least(first:boolean|double|integer|ip|keyword|long|text|version, ?rest...:boolean|double|integer|ip|keyword|long|text|version)"
 "keyword left(string:keyword|text, length:integer)"
 "integer length(string:keyword|text)"
@@ -144,6 +145,7 @@ ends_with     |[str, suffix]                       |["keyword|text", "keyword|te
 floor         |number                              |"double|integer|long|unsigned_long"                                                                                               |Numeric expression. If `null`, the function returns `null`.
 from_base64   |string                              |"keyword|text"                                                                                                                    |A base64 string.
 greatest      |first                               |"boolean|double|integer|ip|keyword|long|text|version"                                                                             |First of the columns to evaluate.
+ip_prefix     |[ip, prefixLengthV4, prefixLengthV6]|[ip, integer, integer]                                                                                                            |[IP address of type `ip` (both IPv4 and IPv6 are supported)., Prefix length for IPv4 addresses., Prefix length for IPv6 addresses.]
 least         |first                               |"boolean|double|integer|ip|keyword|long|text|version"                                                                             |First of the columns to evaluate.
 left          |[string, length]                    |["keyword|text", integer]                                                                                                         |[The string from which to return a substring., The number of characters to return.]
 length        |string                              |"keyword|text"                                                                                                                    |String expression. If `null`, the function returns `null`.
@@ -259,6 +261,7 @@ ends_with     |Returns a boolean that indicates whether a keyword string ends wi
 floor         |Round a number down to the nearest integer.
 from_base64   |Decode a base64 string.
 greatest      |Returns the maximum value from multiple columns. This is similar to <<esql-mv_max>> except it is intended to run on multiple columns at once.
+ip_prefix     |Truncates an IP to a given prefix length.
 least         |Returns the minimum value from multiple columns. This is similar to <<esql-mv_min>> except it is intended to run on multiple columns at once.
 left          |Returns the substring that extracts 'length' chars from 'string' starting from the left.
 length        |Returns the character length of a string.
@@ -375,6 +378,7 @@ ends_with     |boolean
 floor         |"double|integer|long|unsigned_long"                                                                                         |false                       |false           |false
 from_base64   |keyword                                                                                                                     |false                       |false           |false
 greatest      |"boolean|double|integer|ip|keyword|long|text|version"                                                                       |false                       |true            |false
+ip_prefix     |ip                                                                                                                          |[false, false, false]       |false           |false
 least         |"boolean|double|integer|ip|keyword|long|text|version"                                                                       |false                       |true            |false
 left          |keyword                                                                                                                     |[false, false]              |false           |false
 length        |integer                                                                                                                     |false                       |false           |false
@@ -471,5 +475,5 @@ countFunctions#[skip:-8.14.99, reason:BIN added]
 meta functions |  stats  a = count(*), b = count(*), c = count(*) |  mv_expand c;
 
 a:long | b:long | c:long
-106    | 106    | 106
+107    | 107    | 107
 ;
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java
new file mode 100644
index 0000000000000..174df48d5ce62
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java
@@ -0,0 +1,183 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.ip;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import java.util.function.Function;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link IpPrefix}.
+ * This class is generated. Do not edit it.
+ */
+public final class IpPrefixEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator ip;
+
+  private final EvalOperator.ExpressionEvaluator prefixLengthV4;
+
+  private final EvalOperator.ExpressionEvaluator prefixLengthV6;
+
+  private final BytesRef scratch;
+
+  private final DriverContext driverContext;
+
+  public IpPrefixEvaluator(Source source, EvalOperator.ExpressionEvaluator ip,
+      EvalOperator.ExpressionEvaluator prefixLengthV4,
+      EvalOperator.ExpressionEvaluator prefixLengthV6, BytesRef scratch,
+      DriverContext driverContext) {
+    this.warnings = new Warnings(source);
+    this.ip = ip;
+    this.prefixLengthV4 = prefixLengthV4;
+    this.prefixLengthV6 = prefixLengthV6;
+    this.scratch = scratch;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (BytesRefBlock ipBlock = (BytesRefBlock) ip.eval(page)) {
+      try (IntBlock prefixLengthV4Block = (IntBlock) prefixLengthV4.eval(page)) {
+        try (IntBlock prefixLengthV6Block = (IntBlock) prefixLengthV6.eval(page)) {
+          BytesRefVector ipVector = ipBlock.asVector();
+          if (ipVector == null) {
+            return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block, prefixLengthV6Block);
+          }
+          IntVector prefixLengthV4Vector = prefixLengthV4Block.asVector();
+          if (prefixLengthV4Vector == null) {
+            return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block, prefixLengthV6Block);
+          }
+          IntVector prefixLengthV6Vector = prefixLengthV6Block.asVector();
+          if (prefixLengthV6Vector == null) {
+            return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block, prefixLengthV6Block);
+          }
+          return eval(page.getPositionCount(), ipVector, prefixLengthV4Vector, prefixLengthV6Vector);
+        }
+      }
+    }
+  }
+
+  public BytesRefBlock eval(int positionCount, BytesRefBlock ipBlock, IntBlock prefixLengthV4Block,
+      IntBlock prefixLengthV6Block) {
+    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      BytesRef ipScratch = new BytesRef();
+      position: for (int p = 0; p < positionCount; p++) {
+        if (ipBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (ipBlock.getValueCount(p) != 1) {
+          if (ipBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        if (prefixLengthV4Block.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (prefixLengthV4Block.getValueCount(p) != 1) {
+          if (prefixLengthV4Block.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        if (prefixLengthV6Block.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (prefixLengthV6Block.getValueCount(p) != 1) {
+          if (prefixLengthV6Block.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        try {
+          result.appendBytesRef(IpPrefix.process(ipBlock.getBytesRef(ipBlock.getFirstValueIndex(p), ipScratch), prefixLengthV4Block.getInt(prefixLengthV4Block.getFirstValueIndex(p)), prefixLengthV6Block.getInt(prefixLengthV6Block.getFirstValueIndex(p)), scratch));
+        } catch (IllegalArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  public BytesRefBlock eval(int positionCount, BytesRefVector ipVector,
+      IntVector prefixLengthV4Vector, IntVector prefixLengthV6Vector) {
+    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      BytesRef ipScratch = new BytesRef();
+      position: for (int p = 0; p < positionCount; p++) {
+        try {
+          result.appendBytesRef(IpPrefix.process(ipVector.getBytesRef(p, ipScratch), prefixLengthV4Vector.getInt(p), prefixLengthV6Vector.getInt(p), scratch));
+        } catch (IllegalArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "IpPrefixEvaluator[" + "ip=" + ip + ", prefixLengthV4=" + prefixLengthV4 + ", prefixLengthV6=" + prefixLengthV6 + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(ip, prefixLengthV4, prefixLengthV6);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory ip;
+
+    private final EvalOperator.ExpressionEvaluator.Factory prefixLengthV4;
+
+    private final EvalOperator.ExpressionEvaluator.Factory prefixLengthV6;
+
+    private final Function<DriverContext, BytesRef> scratch;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory ip,
+        EvalOperator.ExpressionEvaluator.Factory prefixLengthV4,
+        EvalOperator.ExpressionEvaluator.Factory prefixLengthV6,
+        Function<DriverContext, BytesRef> scratch) {
+      this.source = source;
+      this.ip = ip;
+      this.prefixLengthV4 = prefixLengthV4;
+      this.prefixLengthV6 = prefixLengthV6;
+      this.scratch = scratch;
+    }
+
+    @Override
+    public IpPrefixEvaluator get(DriverContext context) {
+      return new IpPrefixEvaluator(source, ip.get(context), prefixLengthV4.get(context), prefixLengthV6.get(context), scratch.apply(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "IpPrefixEvaluator[" + "ip=" + ip + ", prefixLengthV4=" + prefixLengthV4 + ", prefixLengthV6=" + prefixLengthV6 + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java
new file mode 100644
index 0000000000000..a6cb7c7f9b687
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java
@@ -0,0 +1,148 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.ip;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import java.util.function.Function;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link IpPrefix}.
+ * This class is generated. Do not edit it.
+ */
+public final class IpPrefixOnlyV4Evaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator ip;
+
+  private final EvalOperator.ExpressionEvaluator prefixLengthV4;
+
+  private final BytesRef scratch;
+
+  private final DriverContext driverContext;
+
+  public IpPrefixOnlyV4Evaluator(Source source, EvalOperator.ExpressionEvaluator ip,
+      EvalOperator.ExpressionEvaluator prefixLengthV4, BytesRef scratch,
+      DriverContext driverContext) {
+    this.warnings = new Warnings(source);
+    this.ip = ip;
+    this.prefixLengthV4 = prefixLengthV4;
+    this.scratch = scratch;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (BytesRefBlock ipBlock = (BytesRefBlock) ip.eval(page)) {
+      try (IntBlock prefixLengthV4Block = (IntBlock) prefixLengthV4.eval(page)) {
+        BytesRefVector ipVector = ipBlock.asVector();
+        if (ipVector == null) {
+          return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block);
+        }
+        IntVector prefixLengthV4Vector = prefixLengthV4Block.asVector();
+        if (prefixLengthV4Vector == null) {
+          return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block);
+        }
+        return eval(page.getPositionCount(), ipVector, prefixLengthV4Vector).asBlock();
+      }
+    }
+  }
+
+  public BytesRefBlock eval(int positionCount, BytesRefBlock ipBlock,
+      IntBlock prefixLengthV4Block) {
+    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      BytesRef ipScratch = new BytesRef();
+      position: for (int p = 0; p < positionCount; p++) {
+        if (ipBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (ipBlock.getValueCount(p) != 1) {
+          if (ipBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        if (prefixLengthV4Block.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (prefixLengthV4Block.getValueCount(p) != 1) {
+          if (prefixLengthV4Block.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        result.appendBytesRef(IpPrefix.process(ipBlock.getBytesRef(ipBlock.getFirstValueIndex(p), ipScratch), prefixLengthV4Block.getInt(prefixLengthV4Block.getFirstValueIndex(p)), scratch));
+      }
+      return result.build();
+    }
+  }
+
+  public BytesRefVector eval(int positionCount, BytesRefVector ipVector,
+      IntVector prefixLengthV4Vector) {
+    try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) {
+      BytesRef ipScratch = new BytesRef();
+      position: for (int p = 0; p < positionCount; p++) {
+        result.appendBytesRef(IpPrefix.process(ipVector.getBytesRef(p, ipScratch), prefixLengthV4Vector.getInt(p), scratch));
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "IpPrefixOnlyV4Evaluator[" + "ip=" + ip + ", prefixLengthV4=" + prefixLengthV4 + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(ip, prefixLengthV4);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory ip;
+
+    private final EvalOperator.ExpressionEvaluator.Factory prefixLengthV4;
+
+    private final Function<DriverContext, BytesRef> scratch;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory ip,
+        EvalOperator.ExpressionEvaluator.Factory prefixLengthV4,
+        Function<DriverContext, BytesRef> scratch) {
+      this.source = source;
+      this.ip = ip;
+      this.prefixLengthV4 = prefixLengthV4;
+      this.scratch = scratch;
+    }
+
+    @Override
+    public IpPrefixOnlyV4Evaluator get(DriverContext context) {
+      return new IpPrefixOnlyV4Evaluator(source, ip.get(context), prefixLengthV4.get(context), scratch.apply(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "IpPrefixOnlyV4Evaluator[" + "ip=" + ip + ", prefixLengthV4=" + prefixLengthV4 + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index ca90b9e1e3f2e..e8f136c297ce0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -27,6 +27,11 @@ public class EsqlCapabilities {
      */
     private static final String FN_CBRT = "fn_cbrt";
 
+    /**
+     * Support for function {@code IP_PREFIX}.
+     */
+    private static final String FN_IP_PREFIX = "fn_ip_prefix";
+
     /**
      * Optimization for ST_CENTROID changed some results in cartesian data. #108713
      */
@@ -47,6 +52,7 @@ public class EsqlCapabilities {
     private static Set<String> capabilities() {
         List<String> caps = new ArrayList<>();
         caps.add(FN_CBRT);
+        caps.add(FN_IP_PREFIX);
         caps.add(ST_CENTROID_AGG_OPTIMIZED);
         caps.add(METADATA_IGNORED_FIELD);
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index 90f28b263f07a..382b02d2f9b35 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -52,6 +52,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
+import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Acos;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Asin;
@@ -257,6 +258,7 @@ private FunctionDefinition[][] functions() {
             new FunctionDefinition[] { def(Coalesce.class, Coalesce::new, "coalesce"), },
             // IP
             new FunctionDefinition[] { def(CIDRMatch.class, CIDRMatch::new, "cidr_match") },
+            new FunctionDefinition[] { def(IpPrefix.class, IpPrefix::new, "ip_prefix") },
             // conversion functions
             new FunctionDefinition[] {
                 def(FromBase64.class, FromBase64::new, "from_base64"),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
new file mode 100644
index 0000000000000..d271742aef82d
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
@@ -0,0 +1,191 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.ip;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.ann.Evaluator;
+import org.elasticsearch.compute.ann.Fixed;
+import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.function.OptionalArgument;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isIPAndExact;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
+
+/**
+ * Truncates an IP value to a given prefix length.
+ */
+public class IpPrefix extends EsqlScalarFunction implements OptionalArgument {
+    // Borrowed from Lucene, rfc4291 prefix
+    private static final byte[] IPV4_PREFIX = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1 };
+
+    private final Expression ipField;
+    private final Expression prefixLengthV4Field;
+    private final Expression prefixLengthV6Field;
+
+    @FunctionInfo(
+        returnType = "ip",
+        description = "Truncates an IP to a given prefix length.",
+        examples = @Example(file = "ip", tag = "ipPrefix")
+    )
+    public IpPrefix(
+        Source source,
+        @Param(
+            name = "ip",
+            type = { "ip" },
+            description = "IP address of type `ip` (both IPv4 and IPv6 are supported)."
+        ) Expression ipField,
+        @Param(
+            name = "prefixLengthV4",
+            type = { "integer" },
+            description = "Prefix length for IPv4 addresses."
+        ) Expression prefixLengthV4Field,
+        @Param(
+            name = "prefixLengthV6",
+            type = { "integer" },
+            description = "Prefix length for IPv6 addresses."
+        ) Expression prefixLengthV6Field
+    ) {
+        super(source, Arrays.asList(ipField, prefixLengthV4Field, prefixLengthV6Field));
+        this.ipField = ipField;
+        this.prefixLengthV4Field = prefixLengthV4Field;
+        this.prefixLengthV6Field = prefixLengthV6Field;
+    }
+
+    public static IpPrefix readFrom(PlanStreamInput in) throws IOException {
+        return new IpPrefix(in.readSource(), in.readExpression(), in.readExpression(), in.readExpression());
+    }
+
+    public static void writeTo(PlanStreamOutput out, IpPrefix ipPrefix) throws IOException {
+        out.writeSource(ipPrefix.source());
+        List<Expression> fields = ipPrefix.children();
+        assert fields.size() == 3;
+        out.writeExpression(fields.get(0));
+        out.writeExpression(fields.get(1));
+        out.writeExpression(fields.get(2));
+    }
+
+    public Expression ipField() {
+        return ipField;
+    }
+
+    public Expression prefixLengthV4Field() {
+        return prefixLengthV4Field;
+    }
+
+    public Expression prefixLengthV6Field() {
+        return prefixLengthV6Field;
+    }
+
+    @Override
+    public boolean foldable() {
+        return Expressions.foldable(children());
+    }
+
+    @Override
+    public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
+        var ipEvaluatorSupplier = toEvaluator.apply(ipField);
+        var prefixLengthV4EvaluatorSupplier = toEvaluator.apply(prefixLengthV4Field);
+        var prefixLengthV6EvaluatorSupplier = toEvaluator.apply(prefixLengthV6Field);
+
+        return new IpPrefixEvaluator.Factory(
+            source(),
+            ipEvaluatorSupplier,
+            prefixLengthV4EvaluatorSupplier,
+            prefixLengthV6EvaluatorSupplier,
+            context -> new BytesRef(new byte[16])
+        );
+    }
+
+    @Evaluator(warnExceptions = IllegalArgumentException.class)
+    static BytesRef process(
+        BytesRef ip,
+        int prefixLengthV4,
+        int prefixLengthV6,
+        @Fixed(includeInToString = false, build = true) BytesRef scratch
+    ) {
+        if (prefixLengthV4 < 0 || prefixLengthV4 > 32) {
+            throw new IllegalArgumentException("Prefix length v4 must be in range [0, 32], found " + prefixLengthV4);
+        }
+        if (prefixLengthV6 < 0 || prefixLengthV6 > 128) {
+            throw new IllegalArgumentException("Prefix length v6 must be in range [0, 128], found " + prefixLengthV6);
+        }
+
+        boolean isIpv4 = Arrays.compareUnsigned(ip.bytes, 0, IPV4_PREFIX.length, IPV4_PREFIX, 0, IPV4_PREFIX.length) == 0;
+
+        if (isIpv4) {
+            makePrefix(ip, scratch, 12 + prefixLengthV4 / 8, prefixLengthV4 % 8);
+        } else {
+            makePrefix(ip, scratch, prefixLengthV6 / 8, prefixLengthV6 % 8);
+        }
+
+        return scratch;
+    }
+
+    private static void makePrefix(BytesRef ip, BytesRef scratch, int fullBytes, int remainingBits) {
+        // Copy the first full bytes
+        System.arraycopy(ip.bytes, ip.offset, scratch.bytes, 0, fullBytes);
+
+        // Copy the last byte ignoring the trailing bits
+        if (remainingBits > 0) {
+            byte lastByteMask = (byte) (0xFF << (8 - remainingBits));
+            scratch.bytes[fullBytes] = (byte) (ip.bytes[fullBytes] & lastByteMask);
+        }
+
+        // Copy the last empty bytes
+        if (fullBytes < 16) {
+            Arrays.fill(scratch.bytes, fullBytes + 1, 16, (byte) 0);
+        }
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.IP;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        return isIPAndExact(ipField, sourceText(), FIRST).and(
+            isType(prefixLengthV4Field, dt -> dt == INTEGER, sourceText(), SECOND, "integer")
+        ).and(isType(prefixLengthV6Field, dt -> dt == INTEGER, sourceText(), THIRD, "integer"));
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new IpPrefix(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, IpPrefix::new, ipField, prefixLengthV4Field, prefixLengthV6Field);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index e846360258ebe..67a6a5d4bb98d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -98,6 +98,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
+import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Acos;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Asin;
@@ -394,6 +395,7 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(ScalarFunction.class, DateTrunc.class, PlanNamedTypes::writeDateTrunc, PlanNamedTypes::readDateTrunc),
             of(ScalarFunction.class, E.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar),
             of(ScalarFunction.class, Greatest.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag),
+            of(ScalarFunction.class, IpPrefix.class, IpPrefix::writeTo, IpPrefix::readFrom),
             of(ScalarFunction.class, Least.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag),
             of(ScalarFunction.class, Log.class, PlanNamedTypes::writeLog, PlanNamedTypes::readLog),
             of(ScalarFunction.class, Now.class, PlanNamedTypes::writeNow, PlanNamedTypes::readNow),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
new file mode 100644
index 0000000000000..e46eaea849bb5
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.ip;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
+
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class IpPrefixTests extends AbstractFunctionTestCase {
+    public IpPrefixTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        var suppliers = List.of(
+            // V4
+            new TestCaseSupplier(
+                List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER),
+                () -> new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("1.2.3.4"), DataTypes.IP, "ip"),
+                        new TestCaseSupplier.TypedData(24, DataTypes.INTEGER, "prefixLengthV4"),
+                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 128), DataTypes.INTEGER, "prefixLengthV6")
+                    ),
+                    "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]",
+                    DataTypes.IP,
+                    equalTo(EsqlDataTypeConverter.stringToIP("1.2.3.0"))
+                )
+            ),
+            new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+                var randomIp = randomIp(true);
+                var randomPrefix = randomIntBetween(0, 32);
+                var cidrString = InetAddresses.toCidrString(randomIp, randomPrefix);
+
+                var ipParameter = EsqlDataTypeConverter.stringToIP(NetworkAddress.format(randomIp));
+                var expectedPrefix = EsqlDataTypeConverter.stringToIP(
+                    NetworkAddress.format(InetAddresses.parseIpRangeFromCidr(cidrString).lowerBound())
+                );
+
+                return new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(ipParameter, DataTypes.IP, "ip"),
+                        new TestCaseSupplier.TypedData(randomPrefix, DataTypes.INTEGER, "prefixLengthV4"),
+                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 128), DataTypes.INTEGER, "prefixLengthV6")
+                    ),
+                    "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]",
+                    DataTypes.IP,
+                    equalTo(expectedPrefix)
+                );
+            }),
+
+            // V6
+            new TestCaseSupplier(
+                List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER),
+                () -> new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::ff"), DataTypes.IP, "ip"),
+                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 32), DataTypes.INTEGER, "prefixLengthV4"),
+                        new TestCaseSupplier.TypedData(127, DataTypes.INTEGER, "prefixLengthV6")
+                    ),
+                    "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]",
+                    DataTypes.IP,
+                    equalTo(EsqlDataTypeConverter.stringToIP("::fe"))
+                )
+            ),
+            new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+                var randomIp = randomIp(false);
+                var randomPrefix = randomIntBetween(0, 128);
+                var cidrString = InetAddresses.toCidrString(randomIp, randomPrefix);
+
+                var ipParameter = EsqlDataTypeConverter.stringToIP(NetworkAddress.format(randomIp));
+                var expectedPrefix = EsqlDataTypeConverter.stringToIP(
+                    NetworkAddress.format(InetAddresses.parseIpRangeFromCidr(cidrString).lowerBound())
+                );
+
+                return new TestCaseSupplier.TestCase(
+                    List.of(
+                        new TestCaseSupplier.TypedData(ipParameter, DataTypes.IP, "ip"),
+                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 32), DataTypes.INTEGER, "prefixLengthV4"),
+                        new TestCaseSupplier.TypedData(randomPrefix, DataTypes.INTEGER, "prefixLengthV6")
+                    ),
+                    "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]",
+                    DataTypes.IP,
+                    equalTo(expectedPrefix)
+                );
+            })
+        );
+
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers)));
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new IpPrefix(source, args.get(0), args.get(1), args.size() == 3 ? args.get(2) : null);
+    }
+}

From 829070dfea109f73f3a04f42c0e71e3f37e90f53 Mon Sep 17 00:00:00 2001
From: Andrei Dan <andrei.dan@elastic.co>
Date: Wed, 29 May 2024 16:20:44 +0100
Subject: [PATCH 036/208] Install the apm templates only if DSL available
 (#109166)

The APM module installs component and index templates that make use of
data stream lifecycle. This makes sure the templates are installed only
once the cluster has the data stream lifecycle feature.

Follow-up to https://github.com/elastic/elasticsearch/pull/108860

Fixes #106461
---
 x-pack/plugin/apm-data/build.gradle           |  1 +
 .../apmdata/APMIndexTemplateRegistry.java     | 19 ++++++++++++--
 .../xpack/apmdata/APMPlugin.java              |  9 ++++++-
 .../APMIndexTemplateRegistryTests.java        | 26 +++++++++++++++++--
 4 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/x-pack/plugin/apm-data/build.gradle b/x-pack/plugin/apm-data/build.gradle
index cbd843d227ff4..354be306a0ddd 100644
--- a/x-pack/plugin/apm-data/build.gradle
+++ b/x-pack/plugin/apm-data/build.gradle
@@ -20,6 +20,7 @@ dependencies {
   compileOnly project(path: xpackModule('core'))
   testImplementation project(path: ':x-pack:plugin:stack')
   testImplementation(testArtifact(project(xpackModule('core'))))
+  testImplementation project(':modules:data-streams')
   clusterModules project(':modules:data-streams')
   clusterModules project(':modules:ingest-common')
   clusterModules project(':modules:ingest-geoip')
diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java
index 6ec287fe2b980..04b0257f4180a 100644
--- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java
+++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java
@@ -10,12 +10,15 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
@@ -39,12 +42,15 @@
  */
 public class APMIndexTemplateRegistry extends IndexTemplateRegistry {
     private static final Logger logger = LogManager.getLogger(APMIndexTemplateRegistry.class);
-
+    // this node feature is a redefinition of {@link DataStreamFeatures#DATA_STREAM_LIFECYCLE} and it's meant to avoid adding a
+    // dependency to the data-streams module just for this
+    public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle");
     private final int version;
 
     private final Map<String, ComponentTemplate> componentTemplates;
     private final Map<String, ComposableIndexTemplate> composableIndexTemplates;
     private final List<IngestPipelineConfig> ingestPipelines;
+    private final FeatureService featureService;
     private volatile boolean enabled;
 
     @SuppressWarnings("unchecked")
@@ -53,7 +59,8 @@ public APMIndexTemplateRegistry(
         ClusterService clusterService,
         ThreadPool threadPool,
         Client client,
-        NamedXContentRegistry xContentRegistry
+        NamedXContentRegistry xContentRegistry,
+        FeatureService featureService
     ) {
         super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
 
@@ -78,6 +85,7 @@ public APMIndexTemplateRegistry(
                 Map.Entry<String, Map<String, Object>> pipelineConfig = map.entrySet().iterator().next();
                 return loadIngestPipeline(pipelineConfig.getKey(), version, (List<String>) pipelineConfig.getValue().get("dependencies"));
             }).collect(Collectors.toList());
+            this.featureService = featureService;
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
@@ -105,6 +113,13 @@ protected String getOrigin() {
         return ClientHelper.APM_ORIGIN;
     }
 
+    @Override
+    protected boolean isClusterReady(ClusterChangedEvent event) {
+        // Ensure current version of the components are installed only after versions that support data stream lifecycle
+        // due to the use of the feature in all the `@lifecycle` component templates
+        return featureService.clusterHasFeature(event.state(), DATA_STREAM_LIFECYCLE);
+    }
+
     @Override
     protected boolean requiresMasterNode() {
         return true;
diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java
index f905c17c04b4c..102b0d38461c3 100644
--- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java
+++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java
@@ -48,7 +48,14 @@ public Collection<?> createComponents(PluginServices services) {
         Settings settings = services.environment().settings();
         ClusterService clusterService = services.clusterService();
         registry.set(
-            new APMIndexTemplateRegistry(settings, clusterService, services.threadPool(), services.client(), services.xContentRegistry())
+            new APMIndexTemplateRegistry(
+                settings,
+                clusterService,
+                services.threadPool(),
+                services.client(),
+                services.xContentRegistry(),
+                services.featureService()
+            )
         );
         if (enabled) {
             APMIndexTemplateRegistry registryInstance = registry.get();
diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java
index 8228d7011c9c1..e9f0775836c71 100644
--- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java
+++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.apmdata;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -29,6 +30,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.datastreams.DataStreamFeatures;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.PipelineConfiguration;
@@ -88,7 +90,7 @@ public void createRegistryAndClient() {
         threadPool = new TestThreadPool(this.getClass().getName());
         client = new VerifyingClient(threadPool);
         clusterService = ClusterServiceUtils.createClusterService(threadPool, clusterSettings);
-        FeatureService featureService = new FeatureService(List.of());
+        FeatureService featureService = new FeatureService(List.of(new DataStreamFeatures()));
         stackTemplateRegistryAccessor = new StackTemplateRegistryAccessor(
             new StackTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY, featureService)
         );
@@ -98,7 +100,8 @@ public void createRegistryAndClient() {
             clusterService,
             threadPool,
             client,
-            NamedXContentRegistry.EMPTY
+            NamedXContentRegistry.EMPTY,
+            featureService
         );
         apmIndexTemplateRegistry.setEnabled(true);
     }
@@ -355,6 +358,25 @@ public void testIndexTemplateConventions() throws Exception {
         }
     }
 
+    public void testThatNothingIsInstalledWhenAllNodesAreNotUpdated() {
+        DiscoveryNode updatedNode = DiscoveryNodeUtils.create("updatedNode");
+        DiscoveryNode outdatedNode = DiscoveryNodeUtils.create("outdatedNode", ESTestCase.buildNewFakeTransportAddress(), Version.V_8_10_0);
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .localNodeId("updatedNode")
+            .masterNodeId("updatedNode")
+            .add(updatedNode)
+            .add(outdatedNode)
+            .build();
+
+        client.setVerifier((a, r, l) -> {
+            fail("if some cluster mode are not updated to at least v.8.11.0 nothing should happen");
+            return null;
+        });
+
+        ClusterChangedEvent event = createClusterChangedEvent(Map.of(), Map.of(), nodes);
+        apmIndexTemplateRegistry.clusterChanged(event);
+    }
+
     private Map<String, ComponentTemplate> getIndependentComponentTemplateConfigs() {
         return apmIndexTemplateRegistry.getComponentTemplateConfigs().entrySet().stream().filter(template -> {
             Settings settings = template.getValue().template().settings();

From 0f8c375bc826e61ed916db457f6a208d148dbd41 Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Wed, 29 May 2024 11:22:04 -0400
Subject: [PATCH 037/208] Forward port release notes for v8.13.4 (#108527)

---
 docs/reference/release-notes.asciidoc        |  2 ++
 docs/reference/release-notes/8.13.4.asciidoc | 22 ++++++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 docs/reference/release-notes/8.13.4.asciidoc

diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
index 9fb5bd70e4d29..9b1257e9054c9 100644
--- a/docs/reference/release-notes.asciidoc
+++ b/docs/reference/release-notes.asciidoc
@@ -8,6 +8,7 @@ This section summarizes the changes in each release.
 
 * <<release-notes-8.15.0>>
 * <<release-notes-8.14.0>>
+* <<release-notes-8.13.4>>
 * <<release-notes-8.13.3>>
 * <<release-notes-8.13.2>>
 * <<release-notes-8.13.1>>
@@ -68,6 +69,7 @@ This section summarizes the changes in each release.
 
 include::release-notes/8.15.0.asciidoc[]
 include::release-notes/8.14.0.asciidoc[]
+include::release-notes/8.13.4.asciidoc[]
 include::release-notes/8.13.3.asciidoc[]
 include::release-notes/8.13.2.asciidoc[]
 include::release-notes/8.13.1.asciidoc[]
diff --git a/docs/reference/release-notes/8.13.4.asciidoc b/docs/reference/release-notes/8.13.4.asciidoc
new file mode 100644
index 0000000000000..bf3f2f497d8fc
--- /dev/null
+++ b/docs/reference/release-notes/8.13.4.asciidoc
@@ -0,0 +1,22 @@
+[[release-notes-8.13.4]]
+== {es} version 8.13.4
+
+Also see <<breaking-changes-8.13,Breaking changes in 8.13>>.
+
+[[bug-8.13.4]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fix Bucket ordering for partial reduction in date histogram and histogram aggregation {es-pull}108184[#108184] (issue: {es-issue}108181[#108181])
+
+ES|QL::
+* Fix `BlockHash` `DirectEncoder` {es-pull}108283[#108283] (issue: {es-issue}108268[#108268])
+
+Snapshot/Restore::
+* Ensure necessary security context for s3 bulk deletions {es-pull}108280[#108280] (issue: {es-issue}108049[#108049])
+
+TSDB::
+* Fix tsdb codec when doc-values spread in two blocks {es-pull}108276[#108276]
+
+

From aed23cd6af7f836f08caedb9dc321fbed4ea8fad Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 May 2024 11:30:30 -0400
Subject: [PATCH 038/208] ESQL: Fix compilation

Two PR passing in the night, break each other.
---
 .../esql/expression/function/scalar/ip/IpPrefix.java      | 8 ++++----
 .../xpack/esql/io/stream/PlanNamedTypes.java              | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
index d271742aef82d..5c576101668b3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
@@ -78,12 +78,12 @@ public IpPrefix(
     }
 
     public static IpPrefix readFrom(PlanStreamInput in) throws IOException {
-        return new IpPrefix(in.readSource(), in.readExpression(), in.readExpression(), in.readExpression());
+        return new IpPrefix(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readExpression());
     }
 
-    public static void writeTo(PlanStreamOutput out, IpPrefix ipPrefix) throws IOException {
-        out.writeSource(ipPrefix.source());
-        List<Expression> fields = ipPrefix.children();
+    public void writeTo(PlanStreamOutput out) throws IOException {
+        source().writeTo(out);
+        List<Expression> fields = children();
         assert fields.size() == 3;
         out.writeExpression(fields.get(0));
         out.writeExpression(fields.get(1));
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index 67a6a5d4bb98d..d941b4ef3f40b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -395,7 +395,7 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(ScalarFunction.class, DateTrunc.class, PlanNamedTypes::writeDateTrunc, PlanNamedTypes::readDateTrunc),
             of(ScalarFunction.class, E.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar),
             of(ScalarFunction.class, Greatest.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag),
-            of(ScalarFunction.class, IpPrefix.class, IpPrefix::writeTo, IpPrefix::readFrom),
+            of(ScalarFunction.class, IpPrefix.class, (out, prefix) -> prefix.writeTo(out), IpPrefix::readFrom),
             of(ScalarFunction.class, Least.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag),
             of(ScalarFunction.class, Log.class, PlanNamedTypes::writeLog, PlanNamedTypes::readLog),
             of(ScalarFunction.class, Now.class, PlanNamedTypes::writeNow, PlanNamedTypes::readNow),

From 6bb687bfb25bbbc8cfac50cf5834b20b03dc575a Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 May 2024 11:34:43 -0400
Subject: [PATCH 039/208] ESQL: Drop unused wrapper (#109125)

This isn't used any more.
---
 .../planner/EsqlExpressionTranslators.java     | 18 ++----------------
 1 file changed, 2 insertions(+), 16 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
index d78d2d7f79d80..8873bd770a84e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
@@ -53,7 +53,6 @@
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
-import java.util.function.Supplier;
 
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
@@ -376,26 +375,13 @@ public static void checkSpatialRelatesFunction(Expression constantExpression, Sh
             );
         }
 
-        /**
-         * We should normally be using the real `wrapFunctionQuery` above, so we get the benefits of `SingleValueQuery`,
-         * but at the moment `SingleValueQuery` makes use of `SortDocValues` to determine if the results are single or multi-valued,
-         * and LeafShapeFieldData does not support `SortedBinaryDocValues getBytesValues()`.
-         * Skipping this code path entirely is a temporary workaround while separate work is being done to simplify `SingleValueQuery`
-         * to rather rely on a new method on `LeafFieldData`. This is both for the benefit of the spatial queries, as well as an
-         * improvement overall.
-         * TODO: Remove this method and call the parent method once the SingleValueQuery improvements have been made
-         */
-        public static Query wrapFunctionQuery(Supplier<Query> querySupplier) {
-            return querySupplier.get();
-        }
-
         public static Query doTranslate(SpatialRelatesFunction bc, TranslatorHandler handler) {
             if (bc.left().foldable()) {
                 checkSpatialRelatesFunction(bc.left(), bc.queryRelation());
-                return wrapFunctionQuery(() -> translate(bc, handler, bc.right(), bc.left()));
+                return translate(bc, handler, bc.right(), bc.left());
             } else {
                 checkSpatialRelatesFunction(bc.right(), bc.queryRelation());
-                return wrapFunctionQuery(() -> translate(bc, handler, bc.left(), bc.right()));
+                return translate(bc, handler, bc.left(), bc.right());
             }
         }
 

From 1582c645bf5e89c4271acec6cb7db886a0ca8fb4 Mon Sep 17 00:00:00 2001
From: Pat Whelan <pat.whelan@elastic.co>
Date: Wed, 29 May 2024 11:40:46 -0400
Subject: [PATCH 040/208] [ML] Offload request to generic threadpool (#109104)

buildTextStructureResponse can take a long time, presumably up to 25s,
and is currently blocking the transport thread for that amount of time.
Offloading the request to the generic threadpool will at least let the
transport thread handle other requests concurrently.

Co-authored-by: David Turner <david.turner@elastic.co>
---
 docs/changelog/109104.yaml                    |  6 +++
 .../transport/TextStructExecutor.java         | 47 +++++++++++++++++++
 .../TransportFindFieldStructureAction.java    | 21 +++++----
 .../TransportFindMessageStructureAction.java  | 20 +++++---
 .../TransportFindStructureAction.java         | 26 +++++++---
 5 files changed, 96 insertions(+), 24 deletions(-)
 create mode 100644 docs/changelog/109104.yaml
 create mode 100644 x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java

diff --git a/docs/changelog/109104.yaml b/docs/changelog/109104.yaml
new file mode 100644
index 0000000000000..985cf14bc5952
--- /dev/null
+++ b/docs/changelog/109104.yaml
@@ -0,0 +1,6 @@
+pr: 109104
+summary: Offload request to generic threadpool
+area: Machine Learning
+type: bug
+issues:
+ - 109100
diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java
new file mode 100644
index 0000000000000..89b21c8dfbcf5
--- /dev/null
+++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.textstructure.transport;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.concurrent.ExecutorService;
+
+import static org.elasticsearch.common.util.concurrent.EsExecutors.DIRECT_EXECUTOR_SERVICE;
+
+/**
+ * workaround for https://github.com/elastic/elasticsearch/issues/97916
+ * TODO delete this entire class when we can
+ */
+public class TextStructExecutor {
+    private final ThreadPool threadPool;
+
+    @Inject
+    public TextStructExecutor(ThreadPool threadPool) {
+        this.threadPool = threadPool;
+    }
+
+    /**
+     * when the workaround is removed, change the value in each consuming class's constructor passes to the super constructor from
+     * DIRECT_EXECUTOR_SERVICE back to threadpool.generic() so that we continue to fork off of the transport thread.
+     */
+    ExecutorService handledTransportActionExecutorService() {
+        return DIRECT_EXECUTOR_SERVICE;
+    }
+
+    /**
+     * when the workaround is removed, change the callers of this function to
+     * {@link ActionListener#completeWith(ActionListener, CheckedSupplier)}.
+     */
+    <T> void execute(ActionListener<T> listener, CheckedSupplier<T, Exception> supplier) {
+        threadPool.generic().execute(ActionRunnable.supply(listener, supplier));
+    }
+}
diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java
index 43a990f6f565b..88e60dc3ffd9f 100644
--- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java
+++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.textstructure.transport;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
@@ -31,6 +32,8 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.threadpool.ThreadPool.Names.GENERIC;
+
 public class TransportFindFieldStructureAction extends HandledTransportAction<FindFieldStructureAction.Request, FindStructureResponse> {
 
     private final Client client;
@@ -58,21 +61,18 @@ protected void doExecute(Task task, FindFieldStructureAction.Request request, Ac
             .setFetchSource(true)
             .setQuery(QueryBuilders.existsQuery(request.getField()))
             .setFetchSource(new String[] { request.getField() }, null)
-            .execute(ActionListener.wrap(searchResponse -> {
-                long hitCount = searchResponse.getHits().getHits().length;
+            .execute(listener.delegateFailureAndWrap((delegate, searchResponse) -> {
+                var hitCount = searchResponse.getHits().getHits().length;
                 if (hitCount < AbstractFindStructureRequest.MIN_SAMPLE_LINE_COUNT) {
-                    listener.onFailure(
+                    delegate.onFailure(
                         new IllegalArgumentException("Input contained too few lines [" + hitCount + "] to obtain a meaningful sample")
                     );
                     return;
                 }
-                List<String> messages = getMessages(searchResponse, request.getField());
-                try {
-                    listener.onResponse(buildTextStructureResponse(messages, request));
-                } catch (Exception e) {
-                    listener.onFailure(e);
-                }
-            }, listener::onFailure));
+                var messages = getMessages(searchResponse, request.getField());
+                // As matching a regular expression might take a while, we run in a different thread to avoid blocking the network thread.
+                threadPool.generic().execute(ActionRunnable.supply(delegate, () -> buildTextStructureResponse(messages, request)));
+            }));
     }
 
     private List<String> getMessages(SearchResponse searchResponse, String field) {
@@ -83,6 +83,7 @@ private List<String> getMessages(SearchResponse searchResponse, String field) {
 
     private FindStructureResponse buildTextStructureResponse(List<String> messages, FindFieldStructureAction.Request request)
         throws Exception {
+        assert ThreadPool.assertCurrentThreadPool(GENERIC);
         TextStructureFinderManager structureFinderManager = new TextStructureFinderManager(threadPool.scheduler());
         TextStructureFinder textStructureFinder = structureFinderManager.findTextStructure(
             messages,
diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java
index 79c21b3cea306..d915a7babcbfe 100644
--- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java
+++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java
@@ -19,32 +19,38 @@
 import org.elasticsearch.xpack.textstructure.structurefinder.TextStructureFinderManager;
 import org.elasticsearch.xpack.textstructure.structurefinder.TextStructureOverrides;
 
+import static org.elasticsearch.threadpool.ThreadPool.Names.GENERIC;
+
 public class TransportFindMessageStructureAction extends HandledTransportAction<FindMessageStructureAction.Request, FindStructureResponse> {
 
     private final ThreadPool threadPool;
+    private final TextStructExecutor executor;
 
     @Inject
-    public TransportFindMessageStructureAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool) {
+    public TransportFindMessageStructureAction(
+        TransportService transportService,
+        ActionFilters actionFilters,
+        ThreadPool threadPool,
+        TextStructExecutor executor
+    ) {
         super(
             FindMessageStructureAction.NAME,
             transportService,
             actionFilters,
             FindMessageStructureAction.Request::new,
-            threadPool.generic()
+            executor.handledTransportActionExecutorService()
         );
         this.threadPool = threadPool;
+        this.executor = executor;
     }
 
     @Override
     protected void doExecute(Task task, FindMessageStructureAction.Request request, ActionListener<FindStructureResponse> listener) {
-        try {
-            listener.onResponse(buildTextStructureResponse(request));
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
+        executor.execute(listener, () -> buildTextStructureResponse(request));
     }
 
     private FindStructureResponse buildTextStructureResponse(FindMessageStructureAction.Request request) throws Exception {
+        assert ThreadPool.assertCurrentThreadPool(GENERIC);
         TextStructureFinderManager structureFinderManager = new TextStructureFinderManager(threadPool.scheduler());
         TextStructureFinder textStructureFinder = structureFinderManager.findTextStructure(
             request.getMessages(),
diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java
index 4257a36bc150a..aa546e1747ba4 100644
--- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java
+++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java
@@ -21,26 +21,38 @@
 
 import java.io.InputStream;
 
+import static org.elasticsearch.threadpool.ThreadPool.Names.GENERIC;
+
 public class TransportFindStructureAction extends HandledTransportAction<FindStructureAction.Request, FindStructureResponse> {
 
     private final ThreadPool threadPool;
+    private final TextStructExecutor executor;
 
     @Inject
-    public TransportFindStructureAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool) {
-        super(FindStructureAction.NAME, transportService, actionFilters, FindStructureAction.Request::new, threadPool.generic());
+    public TransportFindStructureAction(
+        TransportService transportService,
+        ActionFilters actionFilters,
+        ThreadPool threadPool,
+        TextStructExecutor executor
+    ) {
+        super(
+            FindStructureAction.NAME,
+            transportService,
+            actionFilters,
+            FindStructureAction.Request::new,
+            executor.handledTransportActionExecutorService()
+        );
         this.threadPool = threadPool;
+        this.executor = executor;
     }
 
     @Override
     protected void doExecute(Task task, FindStructureAction.Request request, ActionListener<FindStructureResponse> listener) {
-        try {
-            listener.onResponse(buildTextStructureResponse(request));
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
+        executor.execute(listener, () -> buildTextStructureResponse(request));
     }
 
     private FindStructureResponse buildTextStructureResponse(FindStructureAction.Request request) throws Exception {
+        assert ThreadPool.assertCurrentThreadPool(GENERIC);
         TextStructureFinderManager structureFinderManager = new TextStructureFinderManager(threadPool.scheduler());
         try (InputStream sampleStream = request.getSample().streamInput()) {
             TextStructureFinder textStructureFinder = structureFinderManager.findTextStructure(

From 1413c67d99e25a35dd7f5a17f86281b8f1dca750 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?=
 <istvan.szabo@elastic.co>
Date: Wed, 29 May 2024 17:43:10 +0200
Subject: [PATCH 041/208] [DOCS] Amends inference reference docs and tutorials
 (#109159)

* [DOCS] Fixes inference tutorial widgets.

* [DOCS] Adds link to notebooks, rearranges sections in PUT inference API docs.
---
 .../inference/put-inference.asciidoc          | 552 +++++++++---------
 .../semantic-search-inference.asciidoc        |  12 +-
 .../infer-api-ingest-pipeline-widget.asciidoc |   2 +-
 .../infer-api-reindex-widget.asciidoc         |   2 +-
 .../infer-api-search-widget.asciidoc          |   2 +-
 5 files changed, 294 insertions(+), 276 deletions(-)

diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc
index cac03afd29562..354cee3f6a990 100644
--- a/docs/reference/inference/put-inference.asciidoc
+++ b/docs/reference/inference/put-inference.asciidoc
@@ -8,10 +8,11 @@ Creates an {infer} endpoint to perform an {infer} task.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
 {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure
-OpenAI, Google AI Studio or Hugging Face. For built-in models and models uploaded though
-Eland, the {infer} APIs offer an alternative way to use and manage trained
-models. However, if you do not plan to use the {infer} APIs to use these models
-or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
+OpenAI, Google AI Studio or Hugging Face. For built-in models and models
+uploaded though Eland, the {infer} APIs offer an alternative way to use and
+manage trained models. However, if you do not plan to use the {infer} APIs to
+use these models or if you want to use non-NLP models, use the
+<<ml-df-trained-models-apis>>.
 
 
 [discrete]
@@ -38,14 +39,14 @@ The create {infer} API enables you to create an {infer} endpoint and configure a
 
 The following services are available through the {infer} API:
 
+* Azure AI Studio
+* Azure OpenAI
 * Cohere
+* Elasticsearch (for built-in models and models uploaded through Eland)
 * ELSER
+* Google AI Studio
 * Hugging Face
 * OpenAI
-* Azure OpenAI
-* Azure AI Studio
-* Elasticsearch (for built-in models and models uploaded through Eland)
-* Google AI Studio
 
 
 [discrete]
@@ -74,82 +75,34 @@ The type of the {infer} task that the model will perform. Available task types:
 (Required, string)
 The type of service supported for the specified task type.
 Available services:
+
+* `azureopenai`: specify the `completion` or `text_embedding` task type to use the Azure OpenAI service.
+* `azureaistudio`: specify the `completion` or `text_embedding` task type to use the Azure AI Studio service.
 * `cohere`: specify the `completion`, `text_embedding` or the `rerank` task type to use the
 Cohere service.
+* `elasticsearch`: specify the `text_embedding` task type to use the E5
+built-in model or text embedding models uploaded by Eland.
 * `elser`: specify the `sparse_embedding` task type to use the ELSER service.
+* `googleaistudio`: specify the `completion` task to use the Google AI Studio service.
 * `hugging_face`: specify the `text_embedding` task type to use the Hugging Face
 service.
 * `openai`: specify the `completion` or `text_embedding` task type to use the
 OpenAI service.
-* `azureopenai`: specify the `completion` or `text_embedding` task type to use the Azure OpenAI service.
-* `azureaistudio`: specify the `completion` or `text_embedding` task type to use the Azure AI Studio service.
-* `elasticsearch`: specify the `text_embedding` task type to use the E5
-built-in model or text embedding models uploaded by Eland.
-* `googleaistudio`: specify the `completion` task to use the Google AI Studio service.
+
 
 `service_settings`::
 (Required, object)
 Settings used to install the {infer} model. These settings are specific to the
 `service` you specified.
 +
-.`service_settings` for the `cohere` service
-[%collapsible%closed]
-=====
-`api_key`:::
-(Required, string)
-A valid API key of your Cohere account. You can find your Cohere API keys or you
-can create a new one
-https://dashboard.cohere.com/api-keys[on the API keys settings page].
-
-IMPORTANT: You need to provide the API key only once, during the {infer} model
-creation. The <<get-inference-api>> does not retrieve your API key. After
-creating the {infer} model, you cannot change the associated API key. If you
-want to use a different API key, delete the {infer} model and recreate it with
-the same name and the updated API key.
-
-`embedding_type`::
-(Optional, string)
-Only for `text_embedding`. Specifies the types of embeddings you want to get
-back. Defaults to `float`.
-Valid values are:
-  * `byte`: use it for signed int8 embeddings (this is a synonym of `int8`).
-  * `float`: use it for the default float embeddings.
-  * `int8`: use it for signed int8 embeddings.
-
-`model_id`::
-(Optional, string)
-The name of the model to use for the {infer} task.
-To review the available `rerank` models, refer to the
-https://docs.cohere.com/reference/rerank-1[Cohere docs].
-
-To review the available `text_embedding` models, refer to the
-https://docs.cohere.com/reference/embed[Cohere docs]. The default value for
-`text_embedding` is `embed-english-v2.0`.
-=====
-+
-.`service_settings` for the `elser` service
+.`service_settings` for the `azureaistudio` service
 [%collapsible%closed]
 =====
-`num_allocations`:::
-(Required, integer)
-The number of model allocations to create. `num_allocations` must not exceed the
-number of available processors per node divided by the `num_threads`.
 
-`num_threads`:::
-(Required, integer)
-The number of threads to use by each model allocation. `num_threads` must not
-exceed the number of available processors per node divided by the number of
-allocations. Must be a power of 2. Max allowed value is 32.
-=====
-+
-.`service_settings` for the `hugging_face` service
-[%collapsible%closed]
-=====
 `api_key`:::
 (Required, string)
-A valid access token of your Hugging Face account. You can find your Hugging
-Face access tokens or you can create a new one
-https://huggingface.co/settings/tokens[on the settings page].
+A valid API key of your Azure AI Studio model deployment.
+This key can be found on the overview page for your deployment in the management section of your https://ai.azure.com/[Azure AI Studio] account.
 
 IMPORTANT: You need to provide the API key only once, during the {infer} model
 creation. The <<get-inference-api>> does not retrieve your API key. After
@@ -157,43 +110,43 @@ creating the {infer} model, you cannot change the associated API key. If you
 want to use a different API key, delete the {infer} model and recreate it with
 the same name and the updated API key.
 
-`url`:::
-(Required, string)
-The URL endpoint to use for the requests.
-=====
-+
-.`service_settings` for the `openai` service
-[%collapsible%closed]
-=====
-`api_key`:::
+`target`:::
 (Required, string)
-A valid API key of your OpenAI account. You can find your OpenAI API keys in
-your OpenAI account under the
-https://platform.openai.com/api-keys[API keys section].
-
-IMPORTANT: You need to provide the API key only once, during the {infer} model
-creation. The <<get-inference-api>> does not retrieve your API key. After
-creating the {infer} model, you cannot change the associated API key. If you
-want to use a different API key, delete the {infer} model and recreate it with
-the same name and the updated API key.
+The target URL of your Azure AI Studio model deployment.
+This can be found on the overview page for your deployment in the management section of your https://ai.azure.com/[Azure AI Studio] account.
 
-`model_id`:::
+`provider`:::
 (Required, string)
-The name of the model to use for the {infer} task. Refer to the
-https://platform.openai.com/docs/guides/embeddings/what-are-embeddings[OpenAI documentation]
-for the list of available text embedding models.
+The model provider for your deployment.
+Note that some providers may support only certain task types.
+Supported providers include:
 
-`organization_id`:::
-(Optional, string)
-The unique identifier of your organization. You can find the Organization ID in
-your OpenAI account under
-https://platform.openai.com/account/organization[**Settings** > **Organizations**].
+* `cohere` - available for `text_embedding` and `completion` task types
+* `databricks` - available for `completion` task type only
+* `meta` - available for `completion` task type only
+* `microsoft_phi` - available for `completion` task type only
+* `mistral` - available for `completion` task type only
+* `openai` - available for `text_embedding` and `completion` task types
 
-`url`:::
-(Optional, string)
-The URL endpoint to use for the requests. Can be changed for testing purposes.
-Defaults to `https://api.openai.com/v1/embeddings`.
+`endpoint_type`:::
+(Required, string)
+One of `token` or `realtime`.
+Specifies the type of endpoint that is used in your model deployment.
+There are https://learn.microsoft.com/en-us/azure/ai-studio/concepts/deployments-overview#billing-for-deploying-and-inferencing-llms-in-azure-ai-studio[two endpoint types available] for deployment through Azure AI Studio.
+"Pay as you go" endpoints are billed per token.
+For these, you must specify `token` for your `endpoint_type`.
+For "real-time" endpoints which are billed per hour of usage, specify `realtime`.
 
+`rate_limit`:::
+(Optional, object)
+By default, the `azureaistudio` service sets the number of requests allowed per minute to `240`.
+This helps to minimize the number of rate limit errors returned from Azure AI Studio.
+To modify this, set the `requests_per_minute` setting of this object in your service settings:
+```
+"rate_limit": {
+    "requests_per_minute": <<number_of_requests>>
+}
+```
 =====
 +
 .`service_settings` for the `azureopenai` service
@@ -230,14 +183,14 @@ We recommend using the https://learn.microsoft.com/en-us/azure/ai-services/opena
 
 =====
 +
-.`service_settings` for the `azureaistudio` service
+.`service_settings` for the `cohere` service
 [%collapsible%closed]
 =====
-
 `api_key`:::
 (Required, string)
-A valid API key of your Azure AI Studio model deployment.
-This key can be found on the overview page for your deployment in the management section of your https://ai.azure.com/[Azure AI Studio] account.
+A valid API key of your Cohere account. You can find your Cohere API keys or you
+can create a new one
+https://dashboard.cohere.com/api-keys[on the API keys settings page].
 
 IMPORTANT: You need to provide the API key only once, during the {infer} model
 creation. The <<get-inference-api>> does not retrieve your API key. After
@@ -245,43 +198,61 @@ creating the {infer} model, you cannot change the associated API key. If you
 want to use a different API key, delete the {infer} model and recreate it with
 the same name and the updated API key.
 
-`target`:::
-(Required, string)
-The target URL of your Azure AI Studio model deployment.
-This can be found on the overview page for your deployment in the management section of your https://ai.azure.com/[Azure AI Studio] account.
+`embedding_type`::
+(Optional, string)
+Only for `text_embedding`. Specifies the types of embeddings you want to get
+back. Defaults to `float`.
+Valid values are:
+  * `byte`: use it for signed int8 embeddings (this is a synonym of `int8`).
+  * `float`: use it for the default float embeddings.
+  * `int8`: use it for signed int8 embeddings.
 
-`provider`:::
+`model_id`::
+(Optional, string)
+The name of the model to use for the {infer} task.
+To review the available `rerank` models, refer to the
+https://docs.cohere.com/reference/rerank-1[Cohere docs].
+
+To review the available `text_embedding` models, refer to the
+https://docs.cohere.com/reference/embed[Cohere docs]. The default value for
+`text_embedding` is `embed-english-v2.0`.
+=====
++
+.`service_settings` for the `elasticsearch` service
+[%collapsible%closed]
+=====
+`model_id`:::
 (Required, string)
-The model provider for your deployment.
-Note that some providers may support only certain task types.
-Supported providers include:
+The name of the model to use for the {infer} task. It can be the
+ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or
+a text embedding model already
+{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
 
-* `openai` - available for `text_embedding` and `completion` task types
-* `mistral` - available for `completion` task type only
-* `meta` - available for `completion` task type only
-* `microsoft_phi` - available for `completion` task type only
-* `cohere` - available for `text_embedding` and `completion` task types
-* `databricks` - available for `completion` task type only
+`num_allocations`:::
+(Required, integer)
+The number of model allocations to create. `num_allocations` must not exceed the
+number of available processors per node divided by the `num_threads`.
 
-`endpoint_type`:::
-(Required, string)
-One of `token` or `realtime`.
-Specifies the type of endpoint that is used in your model deployment.
-There are https://learn.microsoft.com/en-us/azure/ai-studio/concepts/deployments-overview#billing-for-deploying-and-inferencing-llms-in-azure-ai-studio[two endpoint types available] for deployment through Azure AI Studio.
-"Pay as you go" endpoints are billed per token.
-For these, you must specify `token` for your `endpoint_type`.
-For "real-time" endpoints which are billed per hour of usage, specify `realtime`.
+`num_threads`:::
+(Required, integer)
+The number of threads to use by each model allocation. `num_threads` must not
+exceed the number of available processors per node divided by the number of
+allocations. Must be a power of 2. Max allowed value is 32.
+=====
++
+.`service_settings` for the `elser` service
+[%collapsible%closed]
+=====
+`num_allocations`:::
+(Required, integer)
+The number of model allocations to create. `num_allocations` must not exceed the
+number of available processors per node divided by the `num_threads`.
 
-`rate_limit`:::
-(Optional, object)
-By default, the `azureaistudio` service sets the number of requests allowed per minute to `240`.
-This helps to minimize the number of rate limit errors returned from Azure AI Studio.
-To modify this, set the `requests_per_minute` setting of this object in your service settings:
-```
-"rate_limit": {
-    "requests_per_minute": <<number_of_requests>>
-}
-```
+`num_threads`:::
+(Required, integer)
+The number of threads to use by each model allocation. `num_threads` must not
+exceed the number of available processors per node divided by the number of
+allocations. Must be a power of 2. Max allowed value is 32.
 =====
 +
 .`service_settings` for the `googleiastudio` service
@@ -311,26 +282,58 @@ To modify this, set the `requests_per_minute` setting of this object in your ser
 --
 =====
 +
-.`service_settings` for the `elasticsearch` service
+.`service_settings` for the `hugging_face` service
+[%collapsible%closed]
+=====
+`api_key`:::
+(Required, string)
+A valid access token of your Hugging Face account. You can find your Hugging
+Face access tokens or you can create a new one
+https://huggingface.co/settings/tokens[on the settings page].
+
+IMPORTANT: You need to provide the API key only once, during the {infer} model
+creation. The <<get-inference-api>> does not retrieve your API key. After
+creating the {infer} model, you cannot change the associated API key. If you
+want to use a different API key, delete the {infer} model and recreate it with
+the same name and the updated API key.
+
+`url`:::
+(Required, string)
+The URL endpoint to use for the requests.
+=====
++
+.`service_settings` for the `openai` service
 [%collapsible%closed]
 =====
+`api_key`:::
+(Required, string)
+A valid API key of your OpenAI account. You can find your OpenAI API keys in
+your OpenAI account under the
+https://platform.openai.com/api-keys[API keys section].
+
+IMPORTANT: You need to provide the API key only once, during the {infer} model
+creation. The <<get-inference-api>> does not retrieve your API key. After
+creating the {infer} model, you cannot change the associated API key. If you
+want to use a different API key, delete the {infer} model and recreate it with
+the same name and the updated API key.
+
 `model_id`:::
 (Required, string)
-The name of the model to use for the {infer} task. It can be the
-ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or
-a text embedding model already
-{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
+The name of the model to use for the {infer} task. Refer to the
+https://platform.openai.com/docs/guides/embeddings/what-are-embeddings[OpenAI documentation]
+for the list of available text embedding models.
 
-`num_allocations`:::
-(Required, integer)
-The number of model allocations to create. `num_allocations` must not exceed the
-number of available processors per node divided by the `num_threads`.
+`organization_id`:::
+(Optional, string)
+The unique identifier of your organization. You can find the Organization ID in
+your OpenAI account under
+https://platform.openai.com/account/organization[**Settings** > **Organizations**].
+
+`url`:::
+(Optional, string)
+The URL endpoint to use for the requests. Can be changed for testing purposes.
+Defaults to `https://api.openai.com/v1/embeddings`.
 
-`num_threads`:::
-(Required, integer)
-The number of threads to use by each model allocation. `num_threads` must not
-exceed the number of available processors per node divided by the number of
-allocations. Must be a power of 2. Max allowed value is 32.
 =====
 
 `task_settings`::
@@ -341,6 +344,18 @@ Settings to configure the {infer} task. These settings are specific to the
 .`task_settings` for the `completion` task type
 [%collapsible%closed]
 =====
+`do_sample`:::
+(Optional, float)
+For the `azureaistudio` service only.
+Instructs the inference process to perform sampling or not.
+Has not affect unless `temperature` or `top_p` is specified.
+
+`max_new_tokens`:::
+(Optional, integer)
+For the `azureaistudio` service only.
+Provides a hint for the maximum number of output tokens to be generated.
+Defaults to 64.
+
 `user`:::
 (Optional, string)
 For `openai` service only. Specifies the user issuing the request, which can be
@@ -358,18 +373,6 @@ For the `azureaistudio` service only.
 A number in the range of 0.0 to 2.0 that is an alternative value to temperature that causes the model to consider the results of the tokens with nucleus sampling probability.
 Should not be used if `temperature` is specified.
 
-`do_sample`:::
-(Optional, float)
-For the `azureaistudio` service only.
-Instructs the inference process to perform sampling or not.
-Has not affect unless `temperature` or `top_p` is specified.
-
-`max_new_tokens`:::
-(Optional, integer)
-For the `azureaistudio` service only.
-Provides a hint for the maximum number of output tokens to be generated.
-Defaults to 64.
-
 =====
 +
 .`task_settings` for the `rerank` task type
@@ -422,6 +425,100 @@ request, which can be used for abuse detection.
 
 This section contains example API calls for every service type.
 
+[discrete]
+[[inference-example-azureaistudio]]
+===== Azure AI Studio service
+
+The following example shows how to create an {infer} endpoint called
+`azure_ai_studio_embeddings` to perform a `text_embedding` task type.
+Note that we do not specify a model here, as it is defined already via our Azure AI Studio deployment.
+
+The list of embeddings models that you can choose from in your deployment can be found in the https://ai.azure.com/explore/models?selectedTask=embeddings[Azure AI Studio model explorer].
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/azure_ai_studio_embeddings
+{
+    "service": "azureaistudio",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "target": "<target_uri>",
+        "provider": "<model_provider>",
+        "endpoint_type": "<endpoint_type>"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+The next example shows how to create an {infer} endpoint called
+`azure_ai_studio_completion` to perform a `completion` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/completion/azure_ai_studio_completion
+{
+    "service": "azureaistudio",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "target": "<target_uri>",
+        "provider": "<model_provider>",
+        "endpoint_type": "<endpoint_type>"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+The list of chat completion models that you can choose from in your deployment can be found in the https://ai.azure.com/explore/models?selectedTask=chat-completion[Azure AI Studio model explorer].
+
+
+[discrete]
+[[inference-example-azureopenai]]
+===== Azure OpenAI service
+
+The following example shows how to create an {infer} endpoint called
+`azure_openai_embeddings` to perform a `text_embedding` task type.
+Note that we do not specify a model here, as it is defined already via our Azure OpenAI deployment.
+
+The list of embeddings models that you can choose from in your deployment can be found in the https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#embeddings[Azure models documentation].
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/azure_openai_embeddings
+{
+    "service": "azureopenai",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "resource_name": "<resource_name>",
+        "deployment_id": "<deployment_id>",
+        "api_version": "2024-02-01"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+The next example shows how to create an {infer} endpoint called
+`azure_openai_completion` to perform a `completion` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/completion/azure_openai_completion
+{
+    "service": "azureopenai",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "resource_name": "<resource_name>",
+        "deployment_id": "<deployment_id>",
+        "api_version": "2024-02-01"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+The list of chat completion models that you can choose from in your Azure OpenAI deployment can be found at the following places:
+
+* https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-4-and-gpt-4-turbo-models[GPT-4 and GPT-4 Turbo models]
+* https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-35[GPT-3.5]
+
 
 [discrete]
 [[inference-example-cohere]]
@@ -471,7 +568,7 @@ https://docs.cohere.com/docs/elasticsearch-and-cohere#rerank-search-results-with
 
 [discrete]
 [[inference-example-e5]]
-===== E5 via the elasticsearch service
+===== E5 via the `elasticsearch` service
 
 The following example shows how to create an {infer} endpoint called
 `my-e5-model` to perform a `text_embedding` task type.
@@ -500,6 +597,8 @@ further details, refer to the {ml-docs}/ml-nlp-e5.html[E5 model documentation].
 
 The following example shows how to create an {infer} endpoint called
 `my-elser-model` to perform a `sparse_embedding` task type.
+Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more
+info.
 
 [source,console]
 ------------------------------------------------------------
@@ -533,6 +632,27 @@ Example response:
 // NOTCONSOLE
 
 
+[discrete]
+[[inference-example-googleaistudio]]
+===== Google AI Studio service
+
+The following example shows how to create an {infer} endpoint called
+`google_ai_studio_completion` to perform a `completion` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/completion/google_ai_studio_completion
+{
+    "service": "googleaistudio",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "model_id": "<model_id>"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+
 [discrete]
 [[inference-example-hugging-face]]
 ===== Hugging Face service
@@ -637,115 +757,3 @@ PUT _inference/completion/openai-completion
 ------------------------------------------------------------
 // TEST[skip:TBD]
 
-[discrete]
-[[inference-example-azureopenai]]
-===== Azure OpenAI service
-
-The following example shows how to create an {infer} endpoint called
-`azure_openai_embeddings` to perform a `text_embedding` task type.
-Note that we do not specify a model here, as it is defined already via our Azure OpenAI deployment.
-
-The list of embeddings models that you can choose from in your deployment can be found in the https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#embeddings[Azure models documentation].
-
-[source,console]
-------------------------------------------------------------
-PUT _inference/text_embedding/azure_openai_embeddings
-{
-    "service": "azureopenai",
-    "service_settings": {
-        "api_key": "<api_key>",
-        "resource_name": "<resource_name>",
-        "deployment_id": "<deployment_id>",
-        "api_version": "2024-02-01"
-    }
-}
-------------------------------------------------------------
-// TEST[skip:TBD]
-
-The next example shows how to create an {infer} endpoint called
-`azure_openai_completion` to perform a `completion` task type.
-
-[source,console]
-------------------------------------------------------------
-PUT _inference/completion/azure_openai_completion
-{
-    "service": "azureopenai",
-    "service_settings": {
-        "api_key": "<api_key>",
-        "resource_name": "<resource_name>",
-        "deployment_id": "<deployment_id>",
-        "api_version": "2024-02-01"
-    }
-}
-------------------------------------------------------------
-// TEST[skip:TBD]
-
-The list of chat completion models that you can choose from in your Azure OpenAI deployment can be found at the following places:
-
-* https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-4-and-gpt-4-turbo-models[GPT-4 and GPT-4 Turbo models]
-* https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-35[GPT-3.5]
-
-[discrete]
-[[inference-example-azureaistudio]]
-===== Azure AI Studio service
-
-The following example shows how to create an {infer} endpoint called
-`azure_ai_studio_embeddings` to perform a `text_embedding` task type.
-Note that we do not specify a model here, as it is defined already via our Azure AI Studio deployment.
-
-The list of embeddings models that you can choose from in your deployment can be found in the https://ai.azure.com/explore/models?selectedTask=embeddings[Azure AI Studio model explorer].
-
-[source,console]
-------------------------------------------------------------
-PUT _inference/text_embedding/azure_ai_studio_embeddings
-{
-    "service": "azureaistudio",
-    "service_settings": {
-        "api_key": "<api_key>",
-        "target": "<target_uri>",
-        "provider": "<model_provider>",
-        "endpoint_type": "<endpoint_type>"
-    }
-}
-------------------------------------------------------------
-// TEST[skip:TBD]
-
-The next example shows how to create an {infer} endpoint called
-`azure_ai_studio_completion` to perform a `completion` task type.
-
-[source,console]
-------------------------------------------------------------
-PUT _inference/completion/azure_ai_studio_completion
-{
-    "service": "azureaistudio",
-    "service_settings": {
-        "api_key": "<api_key>",
-        "target": "<target_uri>",
-        "provider": "<model_provider>",
-        "endpoint_type": "<endpoint_type>"
-    }
-}
-------------------------------------------------------------
-// TEST[skip:TBD]
-
-The list of chat completion models that you can choose from in your deployment can be found in the https://ai.azure.com/explore/models?selectedTask=chat-completion[Azure AI Studio model explorer].
-
-[discrete]
-[[inference-example-googleaistudio]]
-===== Google AI Studio service
-
-The following example shows how to create an {infer} endpoint called
-`google_ai_studio_completion` to perform a `completion` task type.
-
-[source,console]
-------------------------------------------------------------
-PUT _inference/completion/google_ai_studio_completion
-{
-    "service": "googleaistudio",
-    "service_settings": {
-        "api_key": "<api_key>>",
-        "model_id": "<model_id>"
-    }
-}
-------------------------------------------------------------
-// TEST[skip:TBD]
diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
index e7e16d74764fa..89464d46744b2 100644
--- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
+++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
@@ -114,4 +114,14 @@ provide the query text and the model you have used to create the embeddings.
 NOTE: If you cancelled the reindexing process, you run the query only a part of
 the data which affects the quality of your results.
 
-include::{es-ref-dir}/tab-widgets/inference-api/infer-api-search-widget.asciidoc[]
\ No newline at end of file
+include::{es-ref-dir}/tab-widgets/inference-api/infer-api-search-widget.asciidoc[]
+
+
+[discrete]
+[[infer-interactive-tutorials]]
+==== Interactive tutorials
+
+You can also find tutorials in an interactive Colab notebook format using the
+{es} Python client:
+* https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/inference-cohere.ipynb[Cohere {infer} tutorial notebook]
+* https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/07-inference.ipynb[OpenAI {infer} tutorial notebook]
\ No newline at end of file
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc
index ce0f41524c1a0..80f6da2cf602a 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc
@@ -8,7 +8,7 @@
       Cohere
     </button>
     <button role="tab"
-            aria-selected="true"
+            aria-selected="false"
             aria-controls="infer-api-ingest-hf-tab"
             id="infer-api-ingest-hf">
       HuggingFace
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc
index b3478b2e9edce..a35ea4e3b0207 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc
@@ -8,7 +8,7 @@
       Cohere
     </button>
     <button role="tab"
-            aria-selected="true"
+            aria-selected="false"
             aria-controls="infer-api-reindex-hf-tab"
             id="infer-api-reindex-hf">
       HuggingFace
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc
index d785718f0b288..17b747e86be4a 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc
@@ -8,7 +8,7 @@
       Cohere
     </button>
     <button role="tab"
-            aria-selected="true"
+            aria-selected="false"
             aria-controls="infer-api-search-hf-tab"
             id="infer-api-search-hf">
       HuggingFace

From 60d954f3ec9af0cdec1e5532d1ed3a034240a49e Mon Sep 17 00:00:00 2001
From: Christos Kalkanis <christos.kalkanis@elastic.co>
Date: Wed, 29 May 2024 18:13:43 +0200
Subject: [PATCH 042/208] Universal Profiling: Change profiling-events
 timestamp mapping (#97933)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Switch to 'date_nanos' for nanosecond resolution.

Co-authored-by: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Co-authored-by: Tim Rühsen <tim.ruhsen@elastic.co>
---
 .../profiling/component-template/profiling-events.json       | 4 ++--
 .../resources/data/profiling-events-all.ndjson               | 2 +-
 .../persistence/ProfilingIndexTemplateRegistry.java          | 5 +++--
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json
index fbba399162ee0..70ffedb3f5462 100644
--- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json
+++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json
@@ -38,8 +38,8 @@
           "type": "keyword"
         },
         "@timestamp": {
-          "type": "date",
-          "format": "epoch_second"
+          "type": "date_nanos",
+          "format": "strict_date_optional_time_nanos||epoch_second"
         },
         "host.id": {
           "type": "keyword"
diff --git a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson
index 7211ad54cbcd1..b09817182eb21 100644
--- a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson
+++ b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson
@@ -1,5 +1,5 @@
 {"create": {"_index": "profiling-events-all"}}
-{"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1700504427"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["S07KmaoGhvNte78xwwRbZQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
+{"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["2023-11-20T19:20:27.000000000Z"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["S07KmaoGhvNte78xwwRbZQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
 {"create": {"_index": "profiling-events-all"}}
 {"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["4tB_mGJrj1xVuMFbXVYwGA"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
 {"create": {"_index": "profiling-events-all"}}
diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java
index 86cefd71b2d5f..e486ffd194472 100644
--- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java
+++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java
@@ -50,10 +50,11 @@ public class ProfilingIndexTemplateRegistry extends IndexTemplateRegistry {
     // version 7: Added 'host.type', 'cloud.provider', 'cloud.region' keyword mappings to profiling-hosts
     // version 8: Changed from disabled _source to synthetic _source for profiling-events-* and profiling-metrics
     // version 9: Changed sort order for profiling-events-*
-    public static final int INDEX_TEMPLATE_VERSION = 9;
+    // version 10: changed mapping profiling-events @timestamp to 'date_nanos' from 'date'
+    public static final int INDEX_TEMPLATE_VERSION = 10;
 
     // history for individual indices / index templates. Only bump these for breaking changes that require to create a new index
-    public static final int PROFILING_EVENTS_VERSION = 3;
+    public static final int PROFILING_EVENTS_VERSION = 4;
     public static final int PROFILING_EXECUTABLES_VERSION = 1;
     public static final int PROFILING_METRICS_VERSION = 2;
     public static final int PROFILING_HOSTS_VERSION = 2;

From 6a51c81abe1308a3a02dd6066eb24c94e1bf9f81 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 May 2024 12:32:04 -0400
Subject: [PATCH 043/208] ESQL: Move data types into core (#109162)

This moves all of the new data types declared in `EsqlDataTypes` into
`DataTypes`. It also removes `EsqlDataTypes#types` and makes
`DataTypes#types` return what that used to return. It doesn't modify any
other methods of `EsqlDataTypes`. That's a change for another time.
---
 .../compute/operator/EvalBenchmark.java       |  3 +-
 .../xpack/esql/core/type/DataTypes.java       | 32 ++++++-
 .../xpack/esql/analysis/Analyzer.java         |  4 +-
 .../xpack/esql/analysis/Verifier.java         |  8 +-
 .../esql/expression/EsqlTypeResolutions.java  | 10 +--
 .../function/EsqlFunctionRegistry.java        |  8 +-
 .../function/aggregate/SpatialCentroid.java   | 10 +--
 .../scalar/convert/ToCartesianPoint.java      |  2 +-
 .../scalar/convert/ToCartesianShape.java      |  4 +-
 .../function/scalar/convert/ToDouble.java     |  8 +-
 .../function/scalar/convert/ToGeoPoint.java   |  2 +-
 .../function/scalar/convert/ToGeoShape.java   |  4 +-
 .../function/scalar/convert/ToInteger.java    |  4 +-
 .../function/scalar/convert/ToLong.java       |  6 +-
 .../function/scalar/convert/ToString.java     |  8 +-
 .../scalar/spatial/SpatialContains.java       |  8 +-
 .../scalar/spatial/SpatialDisjoint.java       |  8 +-
 .../scalar/spatial/SpatialIntersects.java     |  8 +-
 .../spatial/SpatialRelatesFunction.java       |  4 +-
 .../scalar/spatial/SpatialWithin.java         |  8 +-
 .../DateTimeArithmeticOperation.java          |  8 +-
 .../predicate/operator/arithmetic/Neg.java    |  4 +-
 .../predicate/operator/comparison/Equals.java |  9 +-
 .../operator/comparison/NotEquals.java        |  9 +-
 .../xpack/esql/io/stream/PlanNamedTypes.java  |  4 +-
 .../optimizer/LocalLogicalPlanOptimizer.java  |  3 +-
 .../xpack/esql/parser/ExpressionBuilder.java  |  4 +-
 .../xpack/esql/planner/AggregateMapper.java   |  4 +-
 .../xpack/esql/planner/PlannerUtils.java      |  6 +-
 .../querydsl/query/SpatialRelatesQuery.java   |  2 +-
 .../esql/type/EsqlDataTypeConverter.java      | 28 +++---
 .../xpack/esql/type/EsqlDataTypeRegistry.java |  7 +-
 .../xpack/esql/type/EsqlDataTypes.java        | 89 ++++---------------
 .../esql/action/EsqlQueryResponseTests.java   |  4 +-
 .../xpack/esql/analysis/AnalyzerTests.java    |  7 +-
 .../xpack/esql/analysis/VerifierTests.java    |  5 +-
 .../function/AbstractFunctionTestCase.java    | 49 +++++-----
 .../expression/function/TestCaseSupplier.java | 24 ++---
 .../AbstractScalarFunctionTestCase.java       | 12 +--
 .../scalar/conditional/CaseTests.java         | 16 ++--
 .../scalar/convert/ToCartesianPointTests.java |  7 +-
 .../scalar/convert/ToCartesianShapeTests.java |  9 +-
 .../scalar/convert/ToDoubleTests.java         |  7 +-
 .../scalar/convert/ToGeoPointTests.java       | 25 ++----
 .../scalar/convert/ToGeoShapeTests.java       | 27 +++---
 .../scalar/convert/ToIntegerTests.java        |  3 +-
 .../function/scalar/convert/ToLongTests.java  |  5 +-
 .../function/scalar/date/DateTruncTests.java  | 13 ++-
 .../function/scalar/math/BucketTests.java     |  5 +-
 .../AbstractMultivalueFunctionTestCase.java   | 23 ++---
 .../scalar/multivalue/MvConcatTests.java      |  4 +-
 .../scalar/multivalue/MvFirstTests.java       |  9 +-
 .../scalar/multivalue/MvLastTests.java        | 21 +----
 .../scalar/multivalue/MvSliceTests.java       | 25 +++---
 .../function/scalar/nulls/CoalesceTests.java  |  8 +-
 .../function/scalar/nulls/IsNotNullTests.java |  2 +-
 .../function/scalar/nulls/IsNullTests.java    |  2 +-
 .../scalar/spatial/SpatialContainsTests.java  |  6 +-
 .../scalar/spatial/SpatialDisjointTests.java  |  6 +-
 .../spatial/SpatialIntersectsTests.java       |  6 +-
 .../scalar/spatial/SpatialWithinTests.java    |  6 +-
 .../function/scalar/string/ConcatTests.java   |  4 +-
 .../function/scalar/string/RLikeTests.java    |  2 +-
 .../function/scalar/string/ReplaceTests.java  |  7 +-
 .../AbstractBinaryOperatorTestCase.java       |  9 +-
 .../operator/arithmetic/AddTests.java         |  5 +-
 .../operator/arithmetic/NegTests.java         | 25 +++---
 .../operator/arithmetic/SubTests.java         | 29 +++---
 .../esql/io/stream/PlanNamedTypesTests.java   |  3 +-
 .../optimizer/LogicalPlanOptimizerTests.java  |  4 +-
 .../optimizer/PhysicalPlanOptimizerTests.java |  6 +-
 .../xpack/esql/parser/ExpressionTests.java    |  4 +-
 .../xpack/esql/planner/EvalMapperTests.java   |  3 +-
 .../esql/tree/EsqlNodeSubclassTests.java      |  4 +-
 .../esql/type/EsqlDataTypeRegistryTests.java  |  6 +-
 75 files changed, 336 insertions(+), 437 deletions(-)

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
index 1fd2e296f0076..50491da4d39aa 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.planner.Layout;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -113,7 +112,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     new EsField("timestamp", DataTypes.DATETIME, Map.of(), true)
                 );
                 yield EvalMapper.toEvaluator(
-                    new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), EsqlDataTypes.TIME_DURATION), timestamp),
+                    new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataTypes.TIME_DURATION), timestamp),
                     layout(timestamp)
                 ).get(driverContext);
             }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java
index 77f143dd3b20b..c28bf96728ad0 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java
@@ -48,8 +48,6 @@ public final class DataTypes {
     public static final DataType IP               = new DataType("ip",                45,                false, false, true);
     // version
     public static final DataType VERSION          = new DataType("version",           Integer.MAX_VALUE, false, false, true);
-    // binary
-    public static final DataType BINARY           = new DataType("binary",            Integer.MAX_VALUE, false, false, true);
     // complex types
     public static final DataType OBJECT           = new DataType("object",            0,                 false, false, false);
     public static final DataType NESTED           = new DataType("nested",            0,                 false, false, false);
@@ -62,6 +60,23 @@ public final class DataTypes {
         false,
         false
     );
+    public static final DataType DATE_PERIOD = new DataType("DATE_PERIOD", null, 3 * Integer.BYTES, false, false, false);
+    public static final DataType TIME_DURATION = new DataType("TIME_DURATION", null, Integer.BYTES + Long.BYTES, false, false, false);
+    public static final DataType GEO_POINT = new DataType("geo_point", Double.BYTES * 2, false, false, true);
+    public static final DataType CARTESIAN_POINT = new DataType("cartesian_point", Double.BYTES * 2, false, false, true);
+    public static final DataType GEO_SHAPE = new DataType("geo_shape", Integer.MAX_VALUE, false, false, true);
+    public static final DataType CARTESIAN_SHAPE = new DataType("cartesian_shape", Integer.MAX_VALUE, false, false, true);
+
+    /**
+     * These are numeric fields labeled as metric counters in time-series indices. Although stored
+     * internally as numeric fields, they represent cumulative metrics and must not be treated as regular
+     * numeric fields. Therefore, we define them differently and separately from their parent numeric field.
+     * These fields are strictly for use in retrieval from indices, rate aggregation, and casting to their
+     * parent numeric type.
+     */
+    public static final DataType COUNTER_LONG = new DataType("counter_long", Long.BYTES, false, false, true);
+    public static final DataType COUNTER_INTEGER = new DataType("counter_integer", Integer.BYTES, false, false, true);
+    public static final DataType COUNTER_DOUBLE = new DataType("counter_double", Double.BYTES, false, false, true);
 
     private static final Collection<DataType> TYPES = Stream.of(
         UNSUPPORTED,
@@ -81,9 +96,18 @@ public final class DataTypes {
         DATETIME,
         IP,
         VERSION,
-        BINARY,
         OBJECT,
-        NESTED
+        NESTED,
+        SOURCE,
+        DATE_PERIOD,
+        TIME_DURATION,
+        GEO_POINT,
+        CARTESIAN_POINT,
+        CARTESIAN_SHAPE,
+        GEO_SHAPE,
+        COUNTER_LONG,
+        COUNTER_INTEGER,
+        COUNTER_DOUBLE
     ).sorted(Comparator.comparing(DataType::typeName)).toList();
 
     private static final Map<String, DataType> NAME_TO_TYPE = TYPES.stream().collect(toUnmodifiableMap(DataType::typeName, t -> t));
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index 082036e32e71f..b95afe2fadeea 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -98,6 +98,8 @@
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
@@ -106,8 +108,6 @@
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
 import static org.elasticsearch.xpack.esql.stats.FeatureMetric.LIMIT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 
 public class Analyzer extends ParameterizedRuleExecutor<LogicalPlan, AnalyzerContext> {
     // marker list of attributes for plans that do not have any concrete fields to return, but have other computed columns to return
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
index a77b613e3d528..3f6e387e661f1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
@@ -383,10 +383,10 @@ public static Failure validateBinaryComparison(BinaryComparison bc) {
         allowed.add(DataTypes.IP);
         allowed.add(DataTypes.DATETIME);
         allowed.add(DataTypes.VERSION);
-        allowed.add(EsqlDataTypes.GEO_POINT);
-        allowed.add(EsqlDataTypes.GEO_SHAPE);
-        allowed.add(EsqlDataTypes.CARTESIAN_POINT);
-        allowed.add(EsqlDataTypes.CARTESIAN_SHAPE);
+        allowed.add(DataTypes.GEO_POINT);
+        allowed.add(DataTypes.GEO_SHAPE);
+        allowed.add(DataTypes.CARTESIAN_POINT);
+        allowed.add(DataTypes.CARTESIAN_SHAPE);
         if (bc instanceof Equals || bc instanceof NotEquals) {
             allowed.add(DataTypes.BOOLEAN);
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java
index 415a46d1a6091..1e2e58886792a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java
@@ -20,10 +20,10 @@
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 
 public class EsqlTypeResolutions {
 
@@ -65,7 +65,7 @@ public static Expression.TypeResolution isExact(Expression e, String operationNa
         GEO_SHAPE.typeName(),
         CARTESIAN_SHAPE.typeName() };
     private static final String[] POINT_TYPE_NAMES = new String[] { GEO_POINT.typeName(), CARTESIAN_POINT.typeName() };
-    private static final String[] NON_SPATIAL_TYPE_NAMES = EsqlDataTypes.types()
+    private static final String[] NON_SPATIAL_TYPE_NAMES = DataTypes.types()
         .stream()
         .filter(EsqlDataTypes::isRepresentable)
         .filter(t -> EsqlDataTypes.isSpatial(t) == false)
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index 382b02d2f9b35..a8ab961fc201a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -124,8 +124,12 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
@@ -134,10 +138,6 @@
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSUPPORTED;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 
 public final class EsqlFunctionRegistry extends FunctionRegistry {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
index f8e038b70c889..0c5db63209b33 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
@@ -16,10 +16,10 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.planner.ToAggregator;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.List;
 
@@ -72,18 +72,18 @@ public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         DataType type = field().dataType();
         if (useDocValues) {
             // When the points are read as doc-values (eg. from the index), feed them into the doc-values aggregator
-            if (type == EsqlDataTypes.GEO_POINT) {
+            if (type == DataTypes.GEO_POINT) {
                 return new SpatialCentroidGeoPointDocValuesAggregatorFunctionSupplier(inputChannels);
             }
-            if (type == EsqlDataTypes.CARTESIAN_POINT) {
+            if (type == DataTypes.CARTESIAN_POINT) {
                 return new SpatialCentroidCartesianPointDocValuesAggregatorFunctionSupplier(inputChannels);
             }
         } else {
             // When the points are read as WKB from source or as point literals, feed them into the source-values aggregator
-            if (type == EsqlDataTypes.GEO_POINT) {
+            if (type == DataTypes.GEO_POINT) {
                 return new SpatialCentroidGeoPointSourceValuesAggregatorFunctionSupplier(inputChannels);
             }
-            if (type == EsqlDataTypes.CARTESIAN_POINT) {
+            if (type == DataTypes.CARTESIAN_POINT) {
                 return new SpatialCentroidCartesianPointSourceValuesAggregatorFunctionSupplier(inputChannels);
             }
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java
index e54c438296f63..5c4208b9f0ce8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java
@@ -20,10 +20,10 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
 
 public class ToCartesianPoint extends AbstractConvertFunction {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java
index e648ea0c8a814..df2c5186ab730 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java
@@ -20,11 +20,11 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_SHAPE;
 
 public class ToCartesianShape extends AbstractConvertFunction {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java
index b39abb51c4587..901257b63ec2b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java
@@ -14,10 +14,10 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.List;
 import java.util.Map;
@@ -44,9 +44,9 @@ public class ToDouble extends AbstractConvertFunction {
         Map.entry(UNSIGNED_LONG, ToDoubleFromUnsignedLongEvaluator.Factory::new),
         Map.entry(LONG, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd
         Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new), // CastIntToDoubleEvaluator would be a candidate, but not MV'd
-        Map.entry(EsqlDataTypes.COUNTER_DOUBLE, (field, source) -> field),
-        Map.entry(EsqlDataTypes.COUNTER_INTEGER, ToDoubleFromIntEvaluator.Factory::new),
-        Map.entry(EsqlDataTypes.COUNTER_LONG, ToDoubleFromLongEvaluator.Factory::new)
+        Map.entry(DataTypes.COUNTER_DOUBLE, (field, source) -> field),
+        Map.entry(DataTypes.COUNTER_INTEGER, ToDoubleFromIntEvaluator.Factory::new),
+        Map.entry(DataTypes.COUNTER_LONG, ToDoubleFromLongEvaluator.Factory::new)
     );
 
     @FunctionInfo(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java
index d64ddbd8e6f0c..122211efbcf10 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java
@@ -20,10 +20,10 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
 
 public class ToGeoPoint extends AbstractConvertFunction {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java
index 67ec2e47008c2..9d7cc08ec6345 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java
@@ -20,11 +20,11 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 
 public class ToGeoShape extends AbstractConvertFunction {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java
index 3ff1a20318238..ea86328a15362 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java
@@ -14,10 +14,10 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.List;
 import java.util.Map;
@@ -45,7 +45,7 @@ public class ToInteger extends AbstractConvertFunction {
         Map.entry(DOUBLE, ToIntegerFromDoubleEvaluator.Factory::new),
         Map.entry(UNSIGNED_LONG, ToIntegerFromUnsignedLongEvaluator.Factory::new),
         Map.entry(LONG, ToIntegerFromLongEvaluator.Factory::new),
-        Map.entry(EsqlDataTypes.COUNTER_INTEGER, (fieldEval, source) -> fieldEval)
+        Map.entry(DataTypes.COUNTER_INTEGER, (fieldEval, source) -> fieldEval)
     );
 
     @FunctionInfo(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java
index f86bf19bdc184..8d00b19f60657 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java
@@ -14,10 +14,10 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.List;
 import java.util.Map;
@@ -45,8 +45,8 @@ public class ToLong extends AbstractConvertFunction {
         Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new),
         Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new),
         Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new), // CastIntToLongEvaluator would be a candidate, but not MV'd
-        Map.entry(EsqlDataTypes.COUNTER_LONG, (field, source) -> field),
-        Map.entry(EsqlDataTypes.COUNTER_INTEGER, ToLongFromIntEvaluator.Factory::new)
+        Map.entry(DataTypes.COUNTER_LONG, (field, source) -> field),
+        Map.entry(DataTypes.COUNTER_INTEGER, ToLongFromIntEvaluator.Factory::new)
     );
 
     @FunctionInfo(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
index ec5542d40c5bd..04318ba4fff64 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
@@ -22,8 +22,12 @@
 import java.util.Map;
 
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
@@ -37,10 +41,6 @@
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.spatialToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 
 public class ToString extends AbstractConvertFunction implements EvaluatorMapper {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
index 161d11478fd2c..729262d4d5870 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
@@ -36,13 +36,13 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2Ds;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.makeGeometryFromLiteral;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 
 /**
  * This is the primary class for supporting the function ST_CONTAINS.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
index 07fe0a3776fe4..f66533ba38162 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
@@ -34,12 +34,12 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 
 /**
  * This is the primary class for supporting the function ST_DISJOINT.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
index 6ac7a2f659ea8..8abc678415e71 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
@@ -34,12 +34,12 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 
 /**
  * This is the primary class for supporting the function ST_INTERSECTS.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
index 69f10cebb636a..d2527c5a8bcd2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
@@ -40,12 +40,12 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.isNull;
 import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 
 public abstract class SpatialRelatesFunction extends BinaryScalarFunction
     implements
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
index 6c10907ada50a..8ca7a40639e1f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
@@ -35,12 +35,12 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 
 /**
  * This is the primary class for supporting the function ST_WITHIN.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
index 0782d298cc809..38f44988d6ac6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
@@ -23,10 +23,10 @@
 import java.util.function.Function;
 
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.isDateTime;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.isNull;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isDateTimeOrTemporal;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
 
@@ -91,7 +91,7 @@ protected TypeResolution checkCompatibility() {
     }
 
     /**
-     * Override this to allow processing literals of type {@link EsqlDataTypes#DATE_PERIOD} when folding constants.
+     * Override this to allow processing literals of type {@link DataTypes#DATE_PERIOD} when folding constants.
      * Used in {@link DateTimeArithmeticOperation#fold()}.
      * @param left the left period
      * @param right the right period
@@ -100,7 +100,7 @@ protected TypeResolution checkCompatibility() {
     abstract Period fold(Period left, Period right);
 
     /**
-     * Override this to allow processing literals of type {@link EsqlDataTypes#TIME_DURATION} when folding constants.
+     * Override this to allow processing literals of type {@link DataTypes#TIME_DURATION} when folding constants.
      * Used in {@link DateTimeArithmeticOperation#fold()}.
      * @param left the left duration
      * @param right the right duration
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
index c8a10668691a6..0847c740805d9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
@@ -26,8 +26,8 @@
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.TIME_DURATION;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
 
 public class Neg extends UnaryScalarFunction {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
index c57c9c3006652..44c7cb39b9a1a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.time.ZoneId;
 import java.util.Map;
@@ -28,10 +27,10 @@ public class Equals extends EsqlBinaryComparison implements Negatable<EsqlBinary
         Map.entry(DataTypes.LONG, EqualsLongsEvaluator.Factory::new),
         Map.entry(DataTypes.UNSIGNED_LONG, EqualsLongsEvaluator.Factory::new),
         Map.entry(DataTypes.DATETIME, EqualsLongsEvaluator.Factory::new),
-        Map.entry(EsqlDataTypes.GEO_POINT, EqualsGeometriesEvaluator.Factory::new),
-        Map.entry(EsqlDataTypes.CARTESIAN_POINT, EqualsGeometriesEvaluator.Factory::new),
-        Map.entry(EsqlDataTypes.GEO_SHAPE, EqualsGeometriesEvaluator.Factory::new),
-        Map.entry(EsqlDataTypes.CARTESIAN_SHAPE, EqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataTypes.GEO_POINT, EqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataTypes.CARTESIAN_POINT, EqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataTypes.GEO_SHAPE, EqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataTypes.CARTESIAN_SHAPE, EqualsGeometriesEvaluator.Factory::new),
         Map.entry(DataTypes.KEYWORD, EqualsKeywordsEvaluator.Factory::new),
         Map.entry(DataTypes.TEXT, EqualsKeywordsEvaluator.Factory::new),
         Map.entry(DataTypes.VERSION, EqualsKeywordsEvaluator.Factory::new),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java
index 2d7828db899ff..788c2396fa626 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.time.ZoneId;
 import java.util.Map;
@@ -28,10 +27,10 @@ public class NotEquals extends EsqlBinaryComparison implements Negatable<EsqlBin
         Map.entry(DataTypes.LONG, NotEqualsLongsEvaluator.Factory::new),
         Map.entry(DataTypes.UNSIGNED_LONG, NotEqualsLongsEvaluator.Factory::new),
         Map.entry(DataTypes.DATETIME, NotEqualsLongsEvaluator.Factory::new),
-        Map.entry(EsqlDataTypes.GEO_POINT, NotEqualsGeometriesEvaluator.Factory::new),
-        Map.entry(EsqlDataTypes.CARTESIAN_POINT, NotEqualsGeometriesEvaluator.Factory::new),
-        Map.entry(EsqlDataTypes.GEO_SHAPE, NotEqualsGeometriesEvaluator.Factory::new),
-        Map.entry(EsqlDataTypes.CARTESIAN_SHAPE, NotEqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataTypes.GEO_POINT, NotEqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataTypes.CARTESIAN_POINT, NotEqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataTypes.GEO_SHAPE, NotEqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataTypes.CARTESIAN_SHAPE, NotEqualsGeometriesEvaluator.Factory::new),
         Map.entry(DataTypes.KEYWORD, NotEqualsKeywordsEvaluator.Factory::new),
         Map.entry(DataTypes.TEXT, NotEqualsKeywordsEvaluator.Factory::new),
         Map.entry(DataTypes.VERSION, NotEqualsKeywordsEvaluator.Factory::new),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index d941b4ef3f40b..09a514e2c14f7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -217,14 +217,14 @@
 import java.util.function.Function;
 
 import static java.util.Map.entry;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
 import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.Entry.of;
 import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader;
 import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToLong;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
 
 /**
  * A utility class that consists solely of static methods that describe how to serialize and
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
index ffed3bf8e9c38..2bcbbec1a8982 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
@@ -44,7 +44,6 @@
 import org.elasticsearch.xpack.esql.planner.AbstractPhysicalOperationProviders;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
@@ -137,7 +136,7 @@ private LogicalPlan missingToNull(LogicalPlan plan, SearchStats stats) {
             else if (plan instanceof Project project) {
                 var projections = project.projections();
                 List<NamedExpression> newProjections = new ArrayList<>(projections.size());
-                Map<DataType, Alias> nullLiteral = Maps.newLinkedHashMapWithExpectedSize(EsqlDataTypes.types().size());
+                Map<DataType, Alias> nullLiteral = Maps.newLinkedHashMapWithExpectedSize(DataTypes.types().size());
 
                 for (NamedExpression projection : projections) {
                     if (projection instanceof FieldAttribute f && stats.exists(f.qualifiedName()) == false) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
index d026ffbeecbe2..a94d449b9e401 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
@@ -78,14 +78,14 @@
 import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.source;
 import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.typedParsing;
 import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.visitList;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.bigIntegerToUnsignedLong;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.parseTemporalAmout;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToIntegral;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.TIME_DURATION;
 
 public abstract class ExpressionBuilder extends IdentifierBuilder {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
index ac60f46ff8d0c..05bc7d6b1bb8a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
@@ -44,8 +44,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
 
 final class AggregateMapper {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
index 7d7150b7f8532..0b4ee46af69ee 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
@@ -244,13 +244,13 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field
         if (dataType == DataTypes.LONG
             || dataType == DataTypes.DATETIME
             || dataType == DataTypes.UNSIGNED_LONG
-            || dataType == EsqlDataTypes.COUNTER_LONG) {
+            || dataType == DataTypes.COUNTER_LONG) {
             return ElementType.LONG;
         }
-        if (dataType == DataTypes.INTEGER || dataType == EsqlDataTypes.COUNTER_INTEGER) {
+        if (dataType == DataTypes.INTEGER || dataType == DataTypes.COUNTER_INTEGER) {
             return ElementType.INT;
         }
-        if (dataType == DataTypes.DOUBLE || dataType == EsqlDataTypes.COUNTER_DOUBLE) {
+        if (dataType == DataTypes.DOUBLE || dataType == DataTypes.COUNTER_DOUBLE) {
             return ElementType.DOUBLE;
         }
         // unsupported fields are passed through as a BytesRef
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
index f02dd63999e27..9b6d392b5c899 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
@@ -40,7 +40,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
 
 public class SpatialRelatesQuery extends Query {
     private final String field;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
index 52031f9a780b8..7ce2d012bf76b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
@@ -59,8 +59,12 @@
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToLong;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToUnsignedLong;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
@@ -77,10 +81,6 @@
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asUnsignedLong;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 
 public class EsqlDataTypeConverter {
 
@@ -146,10 +146,10 @@ public static Converter converterFor(DataType from, DataType to) {
             if (EsqlDataTypes.isSpatial(to)) {
                 return EsqlConverter.STRING_TO_SPATIAL;
             }
-            if (to == EsqlDataTypes.TIME_DURATION) {
+            if (to == DataTypes.TIME_DURATION) {
                 return EsqlConverter.STRING_TO_TIME_DURATION;
             }
-            if (to == EsqlDataTypes.DATE_PERIOD) {
+            if (to == DataTypes.DATE_PERIOD) {
                 return EsqlConverter.STRING_TO_DATE_PERIOD;
             }
         }
@@ -188,15 +188,15 @@ public static TemporalAmount parseTemporalAmount(Object val, DataType expectedTy
         if ((value.isEmpty() || qualifier.isEmpty()) == false) {
             try {
                 TemporalAmount result = parseTemporalAmout(Integer.parseInt(value.toString()), qualifier.toString(), Source.EMPTY);
-                if (EsqlDataTypes.DATE_PERIOD == expectedType && result instanceof Period
-                    || EsqlDataTypes.TIME_DURATION == expectedType && result instanceof Duration) {
+                if (DataTypes.DATE_PERIOD == expectedType && result instanceof Period
+                    || DataTypes.TIME_DURATION == expectedType && result instanceof Duration) {
                     return result;
                 }
-                if (result instanceof Period && expectedType == EsqlDataTypes.TIME_DURATION) {
-                    errorMessage += ", did you mean " + EsqlDataTypes.DATE_PERIOD + "?";
+                if (result instanceof Period && expectedType == DataTypes.TIME_DURATION) {
+                    errorMessage += ", did you mean " + DataTypes.DATE_PERIOD + "?";
                 }
-                if (result instanceof Duration && expectedType == EsqlDataTypes.DATE_PERIOD) {
-                    errorMessage += ", did you mean " + EsqlDataTypes.TIME_DURATION + "?";
+                if (result instanceof Duration && expectedType == DataTypes.DATE_PERIOD) {
+                    errorMessage += ", did you mean " + DataTypes.TIME_DURATION + "?";
                 }
             } catch (NumberFormatException ex) {
                 // wrong pattern
@@ -431,8 +431,8 @@ public static long booleanToUnsignedLong(boolean number) {
 
     public enum EsqlConverter implements Converter {
 
-        STRING_TO_DATE_PERIOD(x -> EsqlDataTypeConverter.parseTemporalAmount(x, EsqlDataTypes.DATE_PERIOD)),
-        STRING_TO_TIME_DURATION(x -> EsqlDataTypeConverter.parseTemporalAmount(x, EsqlDataTypes.TIME_DURATION)),
+        STRING_TO_DATE_PERIOD(x -> EsqlDataTypeConverter.parseTemporalAmount(x, DataTypes.DATE_PERIOD)),
+        STRING_TO_TIME_DURATION(x -> EsqlDataTypeConverter.parseTemporalAmount(x, DataTypes.TIME_DURATION)),
         STRING_TO_CHRONO_FIELD(EsqlDataTypeConverter::stringToChrono),
         STRING_TO_DATETIME(x -> EsqlDataTypeConverter.dateTimeToLong((String) x)),
         STRING_TO_IP(x -> EsqlDataTypeConverter.stringToIP((String) x)),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java
index 0372bc02b0d7e..936aeab569e5f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java
@@ -10,13 +10,14 @@
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypeRegistry;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.util.Collection;
 
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.isDateTime;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isDateTimeOrTemporal;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrDatePeriod;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrTemporalAmount;
@@ -30,7 +31,7 @@ private EsqlDataTypeRegistry() {}
 
     @Override
     public Collection<DataType> dataTypes() {
-        return EsqlDataTypes.types();
+        return DataTypes.types();
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java
index 7c8cf7c0c2a94..893dcfe9103ac 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java
@@ -10,24 +10,20 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
-import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
-import java.util.stream.Stream;
 
 import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toUnmodifiableMap;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.BYTE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.FLOAT;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.HALF_FLOAT;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.NESTED;
@@ -37,78 +33,29 @@
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.SHORT;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.SOURCE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSUPPORTED;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.isNull;
 
 public final class EsqlDataTypes {
 
-    public static final DataType DATE_PERIOD = new DataType("DATE_PERIOD", null, 3 * Integer.BYTES, false, false, false);
-    public static final DataType TIME_DURATION = new DataType("TIME_DURATION", null, Integer.BYTES + Long.BYTES, false, false, false);
-    public static final DataType GEO_POINT = new DataType("geo_point", Double.BYTES * 2, false, false, true);
-    public static final DataType CARTESIAN_POINT = new DataType("cartesian_point", Double.BYTES * 2, false, false, true);
-    public static final DataType GEO_SHAPE = new DataType("geo_shape", Integer.MAX_VALUE, false, false, true);
-    public static final DataType CARTESIAN_SHAPE = new DataType("cartesian_shape", Integer.MAX_VALUE, false, false, true);
-
-    /**
-     * These are numeric fields labeled as metric counters in time-series indices. Although stored
-     * internally as numeric fields, they represent cumulative metrics and must not be treated as regular
-     * numeric fields. Therefore, we define them differently and separately from their parent numeric field.
-     * These fields are strictly for use in retrieval from indices, rate aggregation, and casting to their
-     * parent numeric type.
-     */
-    public static final DataType COUNTER_LONG = new DataType("counter_long", LONG.size(), false, false, LONG.hasDocValues());
-    public static final DataType COUNTER_INTEGER = new DataType("counter_integer", INTEGER.size(), false, false, INTEGER.hasDocValues());
-    public static final DataType COUNTER_DOUBLE = new DataType("counter_double", DOUBLE.size(), false, false, DOUBLE.hasDocValues());
-
-    private static final Collection<DataType> TYPES = Stream.of(
-        BOOLEAN,
-        UNSUPPORTED,
-        NULL,
-        BYTE,
-        SHORT,
-        INTEGER,
-        LONG,
-        DOUBLE,
-        FLOAT,
-        HALF_FLOAT,
-        KEYWORD,
-        TEXT,
-        DATETIME,
-        DATE_PERIOD,
-        TIME_DURATION,
-        IP,
-        OBJECT,
-        NESTED,
-        SCALED_FLOAT,
-        SOURCE,
-        VERSION,
-        UNSIGNED_LONG,
-        GEO_POINT,
-        CARTESIAN_POINT,
-        CARTESIAN_SHAPE,
-        GEO_SHAPE,
-        COUNTER_LONG,
-        COUNTER_INTEGER,
-        COUNTER_DOUBLE
-    ).sorted(Comparator.comparing(DataType::typeName)).toList();
-
-    private static final Map<String, DataType> NAME_TO_TYPE = TYPES.stream().collect(toUnmodifiableMap(DataType::typeName, t -> t));
+    private static final Map<String, DataType> NAME_TO_TYPE = DataTypes.types()
+        .stream()
+        .collect(toUnmodifiableMap(DataType::typeName, t -> t));
 
     private static final Map<String, DataType> ES_TO_TYPE;
 
     static {
-        Map<String, DataType> map = TYPES.stream().filter(e -> e.esType() != null).collect(toMap(DataType::esType, t -> t));
+        Map<String, DataType> map = DataTypes.types().stream().filter(e -> e.esType() != null).collect(toMap(DataType::esType, t -> t));
         // ES calls this 'point', but ESQL calls it 'cartesian_point'
-        map.put("point", CARTESIAN_POINT);
-        map.put("shape", CARTESIAN_SHAPE);
+        map.put("point", DataTypes.CARTESIAN_POINT);
+        map.put("shape", DataTypes.CARTESIAN_SHAPE);
         ES_TO_TYPE = Collections.unmodifiableMap(map);
     }
 
     private static final Map<String, DataType> NAME_OR_ALIAS_TO_TYPE;
     static {
-        Map<String, DataType> map = TYPES.stream().collect(toMap(DataType::typeName, Function.identity()));
+        Map<String, DataType> map = DataTypes.types().stream().collect(toMap(DataType::typeName, Function.identity()));
         map.put("bool", BOOLEAN);
         map.put("int", INTEGER);
         map.put("string", KEYWORD);
@@ -117,10 +64,6 @@ public final class EsqlDataTypes {
 
     private EsqlDataTypes() {}
 
-    public static Collection<DataType> types() {
-        return TYPES;
-    }
-
     public static DataType fromTypeName(String name) {
         return NAME_TO_TYPE.get(name.toLowerCase(Locale.ROOT));
     }
@@ -185,7 +128,7 @@ public static boolean isDateTimeOrTemporal(DataType t) {
     }
 
     public static boolean isTemporalAmount(DataType t) {
-        return t == DATE_PERIOD || t == TIME_DURATION;
+        return t == DataTypes.DATE_PERIOD || t == DataTypes.TIME_DURATION;
     }
 
     public static boolean isNullOrTemporalAmount(DataType t) {
@@ -193,23 +136,23 @@ public static boolean isNullOrTemporalAmount(DataType t) {
     }
 
     public static boolean isNullOrDatePeriod(DataType t) {
-        return t == DATE_PERIOD || isNull(t);
+        return t == DataTypes.DATE_PERIOD || isNull(t);
     }
 
     public static boolean isNullOrTimeDuration(DataType t) {
-        return t == TIME_DURATION || isNull(t);
+        return t == DataTypes.TIME_DURATION || isNull(t);
     }
 
     public static boolean isSpatial(DataType t) {
-        return t == GEO_POINT || t == CARTESIAN_POINT || t == GEO_SHAPE || t == CARTESIAN_SHAPE;
+        return t == DataTypes.GEO_POINT || t == DataTypes.CARTESIAN_POINT || t == DataTypes.GEO_SHAPE || t == DataTypes.CARTESIAN_SHAPE;
     }
 
     public static boolean isSpatialGeo(DataType t) {
-        return t == GEO_POINT || t == GEO_SHAPE;
+        return t == DataTypes.GEO_POINT || t == DataTypes.GEO_SHAPE;
     }
 
     public static boolean isSpatialPoint(DataType t) {
-        return t == GEO_POINT || t == CARTESIAN_POINT;
+        return t == DataTypes.GEO_POINT || t == DataTypes.CARTESIAN_POINT;
     }
 
     /**
@@ -258,6 +201,6 @@ public static DataType getCounterType(String typeName) {
     }
 
     public static boolean isCounterType(DataType dt) {
-        return dt == COUNTER_LONG || dt == COUNTER_INTEGER || dt == COUNTER_DOUBLE;
+        return dt == DataTypes.COUNTER_LONG || dt == DataTypes.COUNTER_INTEGER || dt == DataTypes.COUNTER_DOUBLE;
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
index 1f4331e13db5c..c1fec5d48d23d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
@@ -125,8 +125,8 @@ EsqlQueryResponse randomResponseAsync(boolean columnar, EsqlQueryResponse.Profil
 
     private ColumnInfo randomColumnInfo() {
         DataType type = randomValueOtherThanMany(
-            t -> false == DataTypes.isPrimitive(t) || t == EsqlDataTypes.DATE_PERIOD || t == EsqlDataTypes.TIME_DURATION,
-            () -> randomFrom(EsqlDataTypes.types())
+            t -> false == DataTypes.isPrimitive(t) || t == DataTypes.DATE_PERIOD || t == DataTypes.TIME_DURATION,
+            () -> randomFrom(DataTypes.types())
         );
         type = EsqlDataTypes.widenSmallNumericTypes(type);
         return new ColumnInfo(randomAlphaOfLength(10), type.esType());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index 11ee262c6a682..b770c9b93f4d5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -54,7 +54,6 @@
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.session.EsqlIndexResolver;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -1655,9 +1654,9 @@ public void testCounterTypes() {
             equalTo(Set.of("network.connections", "network.bytes_in", "network.bytes_out", "network.message_in"))
         );
         assertThat(attributes.get("network.connections").dataType(), equalTo(DataTypes.LONG));
-        assertThat(attributes.get("network.bytes_in").dataType(), equalTo(EsqlDataTypes.COUNTER_LONG));
-        assertThat(attributes.get("network.bytes_out").dataType(), equalTo(EsqlDataTypes.COUNTER_LONG));
-        assertThat(attributes.get("network.message_in").dataType(), equalTo(EsqlDataTypes.COUNTER_DOUBLE));
+        assertThat(attributes.get("network.bytes_in").dataType(), equalTo(DataTypes.COUNTER_LONG));
+        assertThat(attributes.get("network.bytes_out").dataType(), equalTo(DataTypes.COUNTER_LONG));
+        assertThat(attributes.get("network.message_in").dataType(), equalTo(DataTypes.COUNTER_DOUBLE));
     }
 
     public void testMissingAttributeException_InChainedEval() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
index 45f923504a46a..4c3a415c5f706 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.parser.TypedParamValue;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -307,7 +308,7 @@ public void testMixedNumericalNonConvertibleTypesInIn() {
     }
 
     public void testUnsignedLongTypeMixInComparisons() {
-        List<String> types = EsqlDataTypes.types()
+        List<String> types = DataTypes.types()
             .stream()
             .filter(dt -> dt.isNumeric() && EsqlDataTypes.isRepresentable(dt) && dt != UNSIGNED_LONG)
             .map(DataType::typeName)
@@ -345,7 +346,7 @@ public void testUnsignedLongTypeMixInComparisons() {
     }
 
     public void testUnsignedLongTypeMixInArithmetics() {
-        List<String> types = EsqlDataTypes.types()
+        List<String> types = DataTypes.types()
             .stream()
             .filter(dt -> dt.isNumeric() && EsqlDataTypes.isRepresentable(dt) && dt != UNSIGNED_LONG)
             .map(DataType::typeName)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
index 840bfece3937d..3b7d36015dcbc 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
@@ -908,14 +908,7 @@ protected static String typeErrorMessage(boolean includeOrdinal, List<Set<DataTy
 
     private static final Map<Set<DataType>, String> NAMED_EXPECTED_TYPES = Map.ofEntries(
         Map.entry(
-            Set.of(
-                EsqlDataTypes.DATE_PERIOD,
-                DataTypes.DOUBLE,
-                DataTypes.INTEGER,
-                DataTypes.LONG,
-                EsqlDataTypes.TIME_DURATION,
-                DataTypes.NULL
-            ),
+            Set.of(DataTypes.DATE_PERIOD, DataTypes.DOUBLE, DataTypes.INTEGER, DataTypes.LONG, DataTypes.TIME_DURATION, DataTypes.NULL),
             "numeric, date_period or time_duration"
         ),
         Map.entry(Set.of(DataTypes.DATETIME, DataTypes.NULL), "datetime"),
@@ -958,13 +951,13 @@ protected static String typeErrorMessage(boolean includeOrdinal, List<Set<DataTy
         // What Add accepts
         Map.entry(
             Set.of(
-                EsqlDataTypes.DATE_PERIOD,
+                DataTypes.DATE_PERIOD,
                 DataTypes.DATETIME,
                 DataTypes.DOUBLE,
                 DataTypes.INTEGER,
                 DataTypes.LONG,
                 DataTypes.NULL,
-                EsqlDataTypes.TIME_DURATION,
+                DataTypes.TIME_DURATION,
                 DataTypes.UNSIGNED_LONG
             ),
             "datetime or numeric"
@@ -987,7 +980,7 @@ protected static String typeErrorMessage(boolean includeOrdinal, List<Set<DataTy
         Map.entry(
             Set.of(
                 DataTypes.BOOLEAN,
-                EsqlDataTypes.COUNTER_INTEGER,
+                DataTypes.COUNTER_INTEGER,
                 DataTypes.DATETIME,
                 DataTypes.DOUBLE,
                 DataTypes.INTEGER,
@@ -1003,8 +996,8 @@ protected static String typeErrorMessage(boolean includeOrdinal, List<Set<DataTy
         Map.entry(
             Set.of(
                 DataTypes.BOOLEAN,
-                EsqlDataTypes.COUNTER_INTEGER,
-                EsqlDataTypes.COUNTER_LONG,
+                DataTypes.COUNTER_INTEGER,
+                DataTypes.COUNTER_LONG,
                 DataTypes.DATETIME,
                 DataTypes.DOUBLE,
                 DataTypes.INTEGER,
@@ -1020,9 +1013,9 @@ protected static String typeErrorMessage(boolean includeOrdinal, List<Set<DataTy
         Map.entry(
             Set.of(
                 DataTypes.BOOLEAN,
-                EsqlDataTypes.COUNTER_DOUBLE,
-                EsqlDataTypes.COUNTER_INTEGER,
-                EsqlDataTypes.COUNTER_LONG,
+                DataTypes.COUNTER_DOUBLE,
+                DataTypes.COUNTER_INTEGER,
+                DataTypes.COUNTER_LONG,
                 DataTypes.DATETIME,
                 DataTypes.DOUBLE,
                 DataTypes.INTEGER,
@@ -1037,10 +1030,10 @@ protected static String typeErrorMessage(boolean includeOrdinal, List<Set<DataTy
         Map.entry(
             Set.of(
                 DataTypes.BOOLEAN,
-                EsqlDataTypes.CARTESIAN_POINT,
+                DataTypes.CARTESIAN_POINT,
                 DataTypes.DATETIME,
                 DataTypes.DOUBLE,
-                EsqlDataTypes.GEO_POINT,
+                DataTypes.GEO_POINT,
                 DataTypes.INTEGER,
                 DataTypes.KEYWORD,
                 DataTypes.LONG,
@@ -1070,8 +1063,8 @@ protected static String typeErrorMessage(boolean includeOrdinal, List<Set<DataTy
                 DataTypes.BOOLEAN,
                 DataTypes.DATETIME,
                 DataTypes.DOUBLE,
-                EsqlDataTypes.GEO_POINT,
-                EsqlDataTypes.GEO_SHAPE,
+                DataTypes.GEO_POINT,
+                DataTypes.GEO_SHAPE,
                 DataTypes.INTEGER,
                 DataTypes.IP,
                 DataTypes.KEYWORD,
@@ -1083,18 +1076,18 @@ protected static String typeErrorMessage(boolean includeOrdinal, List<Set<DataTy
             ),
             "cartesian_point or datetime or geo_point or numeric or string"
         ),
-        Map.entry(Set.of(EsqlDataTypes.GEO_POINT, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL), "geo_point or string"),
-        Map.entry(Set.of(EsqlDataTypes.CARTESIAN_POINT, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL), "cartesian_point or string"),
+        Map.entry(Set.of(DataTypes.GEO_POINT, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL), "geo_point or string"),
+        Map.entry(Set.of(DataTypes.CARTESIAN_POINT, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL), "cartesian_point or string"),
         Map.entry(
-            Set.of(EsqlDataTypes.GEO_POINT, EsqlDataTypes.GEO_SHAPE, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL),
+            Set.of(DataTypes.GEO_POINT, DataTypes.GEO_SHAPE, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL),
             "geo_point or geo_shape or string"
         ),
         Map.entry(
-            Set.of(EsqlDataTypes.CARTESIAN_POINT, EsqlDataTypes.CARTESIAN_SHAPE, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL),
+            Set.of(DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL),
             "cartesian_point or cartesian_shape or string"
         ),
-        Map.entry(Set.of(EsqlDataTypes.GEO_POINT, EsqlDataTypes.CARTESIAN_POINT, DataTypes.NULL), "geo_point or cartesian_point"),
-        Map.entry(Set.of(EsqlDataTypes.DATE_PERIOD, EsqlDataTypes.TIME_DURATION, DataTypes.NULL), "dateperiod or timeduration")
+        Map.entry(Set.of(DataTypes.GEO_POINT, DataTypes.CARTESIAN_POINT, DataTypes.NULL), "geo_point or cartesian_point"),
+        Map.entry(Set.of(DataTypes.DATE_PERIOD, DataTypes.TIME_DURATION, DataTypes.NULL), "dateperiod or timeduration")
     );
 
     // TODO: generate this message dynamically, a la AbstractConvertFunction#supportedTypesNames()?
@@ -1114,7 +1107,7 @@ private static String expectedType(Set<DataType> validTypes) {
     }
 
     protected static Stream<DataType> representable() {
-        return EsqlDataTypes.types().stream().filter(EsqlDataTypes::isRepresentable);
+        return DataTypes.types().stream().filter(EsqlDataTypes::isRepresentable);
     }
 
     protected static DataType[] representableTypes() {
@@ -1609,6 +1602,6 @@ static Version randomVersion() {
      * All string types (keyword, text, match_only_text, etc).
      */
     protected static DataType[] strings() {
-        return EsqlDataTypes.types().stream().filter(DataTypes::isString).toArray(DataType[]::new);
+        return DataTypes.types().stream().filter(DataTypes::isString).toArray(DataType[]::new);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
index 405d96af622ad..9b86482a1ec88 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
@@ -636,7 +636,7 @@ public static void forUnaryDatetime(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link EsqlDataTypes#GEO_POINT}.
+     * Generate positive test cases for a unary function operating on an {@link DataTypes#GEO_POINT}.
      */
     public static void forUnaryGeoPoint(
         List<TestCaseSupplier> suppliers,
@@ -649,7 +649,7 @@ public static void forUnaryGeoPoint(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link EsqlDataTypes#CARTESIAN_POINT}.
+     * Generate positive test cases for a unary function operating on an {@link DataTypes#CARTESIAN_POINT}.
      */
     public static void forUnaryCartesianPoint(
         List<TestCaseSupplier> suppliers,
@@ -662,7 +662,7 @@ public static void forUnaryCartesianPoint(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link EsqlDataTypes#GEO_SHAPE}.
+     * Generate positive test cases for a unary function operating on an {@link DataTypes#GEO_SHAPE}.
      */
     public static void forUnaryGeoShape(
         List<TestCaseSupplier> suppliers,
@@ -675,7 +675,7 @@ public static void forUnaryGeoShape(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link EsqlDataTypes#CARTESIAN_SHAPE}.
+     * Generate positive test cases for a unary function operating on an {@link DataTypes#CARTESIAN_SHAPE}.
      */
     public static void forUnaryCartesianShape(
         List<TestCaseSupplier> suppliers,
@@ -1006,7 +1006,7 @@ public static List<TypedDataSupplier> dateCases() {
 
     public static List<TypedDataSupplier> datePeriodCases() {
         return List.of(
-            new TypedDataSupplier("<zero date period>", () -> Period.ZERO, EsqlDataTypes.DATE_PERIOD, true),
+            new TypedDataSupplier("<zero date period>", () -> Period.ZERO, DataTypes.DATE_PERIOD, true),
             new TypedDataSupplier(
                 "<random date period>",
                 () -> Period.of(
@@ -1014,7 +1014,7 @@ public static List<TypedDataSupplier> datePeriodCases() {
                     ESTestCase.randomIntBetween(-13, 13),
                     ESTestCase.randomIntBetween(-32, 32)
                 ),
-                EsqlDataTypes.DATE_PERIOD,
+                DataTypes.DATE_PERIOD,
                 true
             )
         );
@@ -1022,11 +1022,11 @@ public static List<TypedDataSupplier> datePeriodCases() {
 
     public static List<TypedDataSupplier> timeDurationCases() {
         return List.of(
-            new TypedDataSupplier("<zero time duration>", () -> Duration.ZERO, EsqlDataTypes.TIME_DURATION, true),
+            new TypedDataSupplier("<zero time duration>", () -> Duration.ZERO, DataTypes.TIME_DURATION, true),
             new TypedDataSupplier(
                 "<up to 7 days duration>",
                 () -> Duration.ofMillis(ESTestCase.randomLongBetween(-604800000L, 604800000L)), // plus/minus 7 days
-                EsqlDataTypes.TIME_DURATION,
+                DataTypes.TIME_DURATION,
                 true
             )
         );
@@ -1050,7 +1050,7 @@ public static List<TypedDataSupplier> cartesianShapeCases() {
 
     public static List<TypedDataSupplier> geoPointCases(Supplier<Boolean> hasAlt) {
         return List.of(
-            new TypedDataSupplier("<geo_point>", () -> GEO.asWkb(GeometryTestUtils.randomPoint(hasAlt.get())), EsqlDataTypes.GEO_POINT)
+            new TypedDataSupplier("<geo_point>", () -> GEO.asWkb(GeometryTestUtils.randomPoint(hasAlt.get())), DataTypes.GEO_POINT)
         );
     }
 
@@ -1059,7 +1059,7 @@ public static List<TypedDataSupplier> cartesianPointCases(Supplier<Boolean> hasA
             new TypedDataSupplier(
                 "<cartesian_point>",
                 () -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint(hasAlt.get())),
-                EsqlDataTypes.CARTESIAN_POINT
+                DataTypes.CARTESIAN_POINT
             )
         );
     }
@@ -1069,7 +1069,7 @@ public static List<TypedDataSupplier> geoShapeCases(Supplier<Boolean> hasAlt) {
             new TypedDataSupplier(
                 "<geo_shape>",
                 () -> GEO.asWkb(GeometryTestUtils.randomGeometryWithoutCircle(0, hasAlt.get())),
-                EsqlDataTypes.GEO_SHAPE
+                DataTypes.GEO_SHAPE
             )
         );
     }
@@ -1079,7 +1079,7 @@ public static List<TypedDataSupplier> cartesianShapeCases(Supplier<Boolean> hasA
             new TypedDataSupplier(
                 "<cartesian_shape>",
                 () -> CARTESIAN.asWkb(ShapeTestUtils.randomGeometry(hasAlt.get())),
-                EsqlDataTypes.CARTESIAN_SHAPE
+                DataTypes.CARTESIAN_SHAPE
             )
         );
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java
index 4422b83751522..18957a7f115b8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java
@@ -80,26 +80,26 @@ public Set<DataType> sortedTypesSet(DataType[] validTypes, DataType... additiona
      * All integer types (long, int, short, byte). For passing to {@link #required} or {@link #optional}.
      */
     protected static DataType[] integers() {
-        return EsqlDataTypes.types().stream().filter(DataType::isInteger).toArray(DataType[]::new);
+        return DataTypes.types().stream().filter(DataType::isInteger).toArray(DataType[]::new);
     }
 
     /**
      * All rational types (double, float, whatever). For passing to {@link #required} or {@link #optional}.
      */
     protected static DataType[] rationals() {
-        return EsqlDataTypes.types().stream().filter(DataType::isRational).toArray(DataType[]::new);
+        return DataTypes.types().stream().filter(DataType::isRational).toArray(DataType[]::new);
     }
 
     /**
      * All numeric types (integers and rationals.) For passing to {@link #required} or {@link #optional}.
      */
     protected static DataType[] numerics() {
-        return EsqlDataTypes.types().stream().filter(DataType::isNumeric).toArray(DataType[]::new);
+        return DataTypes.types().stream().filter(DataType::isNumeric).toArray(DataType[]::new);
     }
 
     protected final DataType[] representableNumerics() {
         // TODO numeric should only include representable numbers but that is a change for a followup
-        return EsqlDataTypes.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new);
+        return DataTypes.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new);
     }
 
     protected record ArgumentSpec(boolean optional, Set<DataType> validTypes) {}
@@ -107,7 +107,7 @@ protected record ArgumentSpec(boolean optional, Set<DataType> validTypes) {}
     public final void testResolveType() {
         List<ArgumentSpec> specs = argSpec();
         for (int mutArg = 0; mutArg < specs.size(); mutArg++) {
-            for (DataType mutArgType : EsqlDataTypes.types()) {
+            for (DataType mutArgType : DataTypes.types()) {
                 List<Expression> args = new ArrayList<>(specs.size());
                 for (int arg = 0; arg < specs.size(); arg++) {
                     if (mutArg == arg) {
@@ -180,7 +180,7 @@ private String expectedTypeName(Set<DataType> validTypes) {
         if (withoutNull.equals(List.of(DataTypes.IP))) {
             return "ip";
         }
-        List<DataType> negations = Stream.concat(Stream.of(numerics()), Stream.of(EsqlDataTypes.DATE_PERIOD, EsqlDataTypes.TIME_DURATION))
+        List<DataType> negations = Stream.concat(Stream.of(numerics()), Stream.of(DataTypes.DATE_PERIOD, DataTypes.TIME_DURATION))
             .sorted(Comparator.comparing(DataType::name))
             .toList();
         if (withoutNull.equals(negations)) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
index 2011dddd986e4..e14b69d627f2e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
@@ -175,30 +175,30 @@ public static Iterable<Object[]> parameters() {
                     DataTypes.VERSION,
                     nullValue()
                 );
-            }), new TestCaseSupplier("cartesian_point", List.of(DataTypes.BOOLEAN, EsqlDataTypes.CARTESIAN_POINT), () -> {
-                BytesRef value = (BytesRef) randomLiteral(EsqlDataTypes.CARTESIAN_POINT).value();
+            }), new TestCaseSupplier("cartesian_point", List.of(DataTypes.BOOLEAN, DataTypes.CARTESIAN_POINT), () -> {
+                BytesRef value = (BytesRef) randomLiteral(DataTypes.CARTESIAN_POINT).value();
                 List<TestCaseSupplier.TypedData> typedData = List.of(
                     new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, EsqlDataTypes.CARTESIAN_POINT, "trueValue")
+                    new TestCaseSupplier.TypedData(value, DataTypes.CARTESIAN_POINT, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=BYTES_REF, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    EsqlDataTypes.CARTESIAN_POINT,
+                    DataTypes.CARTESIAN_POINT,
                     nullValue()
                 );
-            }), new TestCaseSupplier("geo_point", List.of(DataTypes.BOOLEAN, EsqlDataTypes.GEO_POINT), () -> {
-                BytesRef value = (BytesRef) randomLiteral(EsqlDataTypes.GEO_POINT).value();
+            }), new TestCaseSupplier("geo_point", List.of(DataTypes.BOOLEAN, DataTypes.GEO_POINT), () -> {
+                BytesRef value = (BytesRef) randomLiteral(DataTypes.GEO_POINT).value();
                 List<TestCaseSupplier.TypedData> typedData = List.of(
                     new TestCaseSupplier.TypedData(true, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, EsqlDataTypes.GEO_POINT, "trueValue")
+                    new TestCaseSupplier.TypedData(value, DataTypes.GEO_POINT, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=BYTES_REF, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    EsqlDataTypes.GEO_POINT,
+                    DataTypes.GEO_POINT,
                     equalTo(value)
                 );
             }))
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
index d28dca0d7c537..6e63a14a0ad7c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -41,12 +40,12 @@ public static Iterable<Object[]> parameters() {
         final Function<String, String> evaluatorName = s -> "ToCartesianPoint" + s + "Evaluator[field=" + attribute + "]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryCartesianPoint(suppliers, attribute, EsqlDataTypes.CARTESIAN_POINT, v -> v, List.of());
+        TestCaseSupplier.forUnaryCartesianPoint(suppliers, attribute, DataTypes.CARTESIAN_POINT, v -> v, List.of());
         // random strings that don't look like a cartesian point
         TestCaseSupplier.forUnaryStrings(
             suppliers,
             evaluatorName.apply("FromString"),
-            EsqlDataTypes.CARTESIAN_POINT,
+            DataTypes.CARTESIAN_POINT,
             bytesRef -> null,
             bytesRef -> {
                 var exception = expectThrows(Exception.class, () -> CARTESIAN.wktToWkb(bytesRef.utf8ToString()));
@@ -68,7 +67,7 @@ public static Iterable<Object[]> parameters() {
                         dt
                     )
                 ),
-                EsqlDataTypes.CARTESIAN_POINT,
+                DataTypes.CARTESIAN_POINT,
                 bytesRef -> CARTESIAN.wktToWkb(((BytesRef) bytesRef).utf8ToString()),
                 List.of()
             );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
index 0f57ea34381f1..9ffb8bd82a0a8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -41,13 +40,13 @@ public static Iterable<Object[]> parameters() {
         final Function<String, String> evaluatorName = s -> "ToCartesianShape" + s + "Evaluator[field=" + attribute + "]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryCartesianPoint(suppliers, attribute, EsqlDataTypes.CARTESIAN_SHAPE, v -> v, List.of());
-        TestCaseSupplier.forUnaryCartesianShape(suppliers, attribute, EsqlDataTypes.CARTESIAN_SHAPE, v -> v, List.of());
+        TestCaseSupplier.forUnaryCartesianPoint(suppliers, attribute, DataTypes.CARTESIAN_SHAPE, v -> v, List.of());
+        TestCaseSupplier.forUnaryCartesianShape(suppliers, attribute, DataTypes.CARTESIAN_SHAPE, v -> v, List.of());
         // random strings that don't look like a cartesian shape
         TestCaseSupplier.forUnaryStrings(
             suppliers,
             evaluatorName.apply("FromString"),
-            EsqlDataTypes.CARTESIAN_SHAPE,
+            DataTypes.CARTESIAN_SHAPE,
             bytesRef -> null,
             bytesRef -> {
                 var exception = expectThrows(Exception.class, () -> CARTESIAN.wktToWkb(bytesRef.utf8ToString()));
@@ -69,7 +68,7 @@ public static Iterable<Object[]> parameters() {
                         dt
                     )
                 ),
-                EsqlDataTypes.CARTESIAN_SHAPE,
+                DataTypes.CARTESIAN_SHAPE,
                 bytesRef -> CARTESIAN.wktToWkb(((BytesRef) bytesRef).utf8ToString()),
                 List.of()
             );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
index 4d1d4b909e72f..24f7952c67f6a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.math.BigInteger;
 import java.util.ArrayList;
@@ -118,7 +117,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.unary(
             suppliers,
             "Attribute[channel=0]",
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomDouble, EsqlDataTypes.COUNTER_DOUBLE)),
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomDouble, DataTypes.COUNTER_DOUBLE)),
             DataTypes.DOUBLE,
             l -> l,
             List.of()
@@ -126,7 +125,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.unary(
             suppliers,
             evaluatorName.apply("Integer"),
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", () -> randomInt(1000), EsqlDataTypes.COUNTER_INTEGER)),
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", () -> randomInt(1000), DataTypes.COUNTER_INTEGER)),
             DataTypes.DOUBLE,
             l -> ((Integer) l).doubleValue(),
             List.of()
@@ -134,7 +133,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.unary(
             suppliers,
             evaluatorName.apply("Long"),
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", () -> randomLongBetween(1, 1000), EsqlDataTypes.COUNTER_LONG)),
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", () -> randomLongBetween(1, 1000), DataTypes.COUNTER_LONG)),
             DataTypes.DOUBLE,
             l -> ((Long) l).doubleValue(),
             List.of()
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
index c22c48767aafb..4b7d78980a440 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -41,21 +40,15 @@ public static Iterable<Object[]> parameters() {
         final Function<String, String> evaluatorName = s -> "ToGeoPoint" + s + "Evaluator[field=" + attribute + "]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryGeoPoint(suppliers, attribute, EsqlDataTypes.GEO_POINT, v -> v, List.of());
+        TestCaseSupplier.forUnaryGeoPoint(suppliers, attribute, DataTypes.GEO_POINT, v -> v, List.of());
         // random strings that don't look like a geo point
-        TestCaseSupplier.forUnaryStrings(
-            suppliers,
-            evaluatorName.apply("FromString"),
-            EsqlDataTypes.GEO_POINT,
-            bytesRef -> null,
-            bytesRef -> {
-                var exception = expectThrows(Exception.class, () -> GEO.wktToWkb(bytesRef.utf8ToString()));
-                return List.of(
-                    "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
-                    "Line -1:-1: " + exception
-                );
-            }
-        );
+        TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("FromString"), DataTypes.GEO_POINT, bytesRef -> null, bytesRef -> {
+            var exception = expectThrows(Exception.class, () -> GEO.wktToWkb(bytesRef.utf8ToString()));
+            return List.of(
+                "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
+                "Line -1:-1: " + exception
+            );
+        });
         // strings that are geo point representations
         for (DataType dt : List.of(DataTypes.KEYWORD, DataTypes.TEXT)) {
             TestCaseSupplier.unary(
@@ -68,7 +61,7 @@ public static Iterable<Object[]> parameters() {
                         dt
                     )
                 ),
-                EsqlDataTypes.GEO_POINT,
+                DataTypes.GEO_POINT,
                 bytesRef -> GEO.wktToWkb(((BytesRef) bytesRef).utf8ToString()),
                 List.of()
             );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
index e7c29be3b0084..0b1f01ee24183 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -41,22 +40,16 @@ public static Iterable<Object[]> parameters() {
         final Function<String, String> evaluatorName = s -> "ToGeoShape" + s + "Evaluator[field=" + attribute + "]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryGeoPoint(suppliers, attribute, EsqlDataTypes.GEO_SHAPE, v -> v, List.of());
-        TestCaseSupplier.forUnaryGeoShape(suppliers, attribute, EsqlDataTypes.GEO_SHAPE, v -> v, List.of());
+        TestCaseSupplier.forUnaryGeoPoint(suppliers, attribute, DataTypes.GEO_SHAPE, v -> v, List.of());
+        TestCaseSupplier.forUnaryGeoShape(suppliers, attribute, DataTypes.GEO_SHAPE, v -> v, List.of());
         // random strings that don't look like a geo shape
-        TestCaseSupplier.forUnaryStrings(
-            suppliers,
-            evaluatorName.apply("FromString"),
-            EsqlDataTypes.GEO_SHAPE,
-            bytesRef -> null,
-            bytesRef -> {
-                var exception = expectThrows(Exception.class, () -> GEO.wktToWkb(bytesRef.utf8ToString()));
-                return List.of(
-                    "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
-                    "Line -1:-1: " + exception
-                );
-            }
-        );
+        TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("FromString"), DataTypes.GEO_SHAPE, bytesRef -> null, bytesRef -> {
+            var exception = expectThrows(Exception.class, () -> GEO.wktToWkb(bytesRef.utf8ToString()));
+            return List.of(
+                "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
+                "Line -1:-1: " + exception
+            );
+        });
         // strings that are geo_shape representations
         for (DataType dt : List.of(DataTypes.KEYWORD, DataTypes.TEXT)) {
             TestCaseSupplier.unary(
@@ -69,7 +62,7 @@ public static Iterable<Object[]> parameters() {
                         dt
                     )
                 ),
-                EsqlDataTypes.GEO_SHAPE,
+                DataTypes.GEO_SHAPE,
                 bytesRef -> GEO.wktToWkb(((BytesRef) bytesRef).utf8ToString()),
                 List.of()
             );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
index 0dc66fca7dd1d..ffac75f1ac338 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.math.BigInteger;
 import java.util.ArrayList;
@@ -263,7 +262,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.unary(
             suppliers,
             "Attribute[channel=0]",
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, EsqlDataTypes.COUNTER_INTEGER)),
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, DataTypes.COUNTER_INTEGER)),
             DataTypes.INTEGER,
             l -> l,
             List.of()
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
index 77ec9b4ed713b..adb2d68cf526a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.math.BigInteger;
 import java.time.Instant;
@@ -215,7 +214,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.unary(
             suppliers,
             "Attribute[channel=0]",
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomNonNegativeLong, EsqlDataTypes.COUNTER_LONG)),
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomNonNegativeLong, DataTypes.COUNTER_LONG)),
             DataTypes.LONG,
             l -> l,
             List.of()
@@ -223,7 +222,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.unary(
             suppliers,
             evaluatorName.apply("Integer"),
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, EsqlDataTypes.COUNTER_INTEGER)),
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, DataTypes.COUNTER_INTEGER)),
             DataTypes.LONG,
             l -> ((Integer) l).longValue(),
             List.of()
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
index 4adf32479f10c..f33f8d4ba1041 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -140,10 +139,10 @@ public void testDateTruncFunction() {
 
     private static TestCaseSupplier ofDatePeriod(Period period, long value, String expectedDate) {
         return new TestCaseSupplier(
-            List.of(EsqlDataTypes.DATE_PERIOD, DataTypes.DATETIME),
+            List.of(DataTypes.DATE_PERIOD, DataTypes.DATETIME),
             () -> new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(period, EsqlDataTypes.DATE_PERIOD, "interval"),
+                    new TestCaseSupplier.TypedData(period, DataTypes.DATE_PERIOD, "interval"),
                     new TestCaseSupplier.TypedData(value, DataTypes.DATETIME, "date")
                 ),
                 "DateTruncEvaluator[date=Attribute[channel=1], interval=Attribute[channel=0]]",
@@ -155,10 +154,10 @@ private static TestCaseSupplier ofDatePeriod(Period period, long value, String e
 
     private static TestCaseSupplier ofDuration(Duration duration, long value, String expectedDate) {
         return new TestCaseSupplier(
-            List.of(EsqlDataTypes.TIME_DURATION, DataTypes.DATETIME),
+            List.of(DataTypes.TIME_DURATION, DataTypes.DATETIME),
             () -> new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(duration, EsqlDataTypes.TIME_DURATION, "interval"),
+                    new TestCaseSupplier.TypedData(duration, DataTypes.TIME_DURATION, "interval"),
                     new TestCaseSupplier.TypedData(value, DataTypes.DATETIME, "date")
                 ),
                 "DateTruncEvaluator[date=Attribute[channel=1], interval=Attribute[channel=0]]",
@@ -169,7 +168,7 @@ private static TestCaseSupplier ofDuration(Duration duration, long value, String
     }
 
     private static TestCaseSupplier randomSecond() {
-        return new TestCaseSupplier("random second", List.of(EsqlDataTypes.TIME_DURATION, DataTypes.DATETIME), () -> {
+        return new TestCaseSupplier("random second", List.of(DataTypes.TIME_DURATION, DataTypes.DATETIME), () -> {
             String dateFragment = randomIntBetween(2000, 2050)
                 + "-"
                 + pad(randomIntBetween(1, 12))
@@ -183,7 +182,7 @@ private static TestCaseSupplier randomSecond() {
                 + pad(randomIntBetween(0, 59));
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(Duration.ofSeconds(1), EsqlDataTypes.TIME_DURATION, "interval"),
+                    new TestCaseSupplier.TypedData(Duration.ofSeconds(1), DataTypes.TIME_DURATION, "interval"),
                     new TestCaseSupplier.TypedData(toMillis(dateFragment + ".38Z"), DataTypes.DATETIME, "date")
                 ),
                 "DateTruncEvaluator[date=Attribute[channel=1], interval=Attribute[channel=0]]",
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java
index ba4c43b5020c0..b9b672a2111c0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.hamcrest.Matcher;
 
 import java.time.Duration;
@@ -45,7 +44,7 @@ public static Iterable<Object[]> parameters() {
             suppliers,
             "fixed date with period",
             () -> DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00.00Z"),
-            EsqlDataTypes.DATE_PERIOD,
+            DataTypes.DATE_PERIOD,
             Period.ofYears(1),
             "[YEAR_OF_CENTURY in Z][fixed to midnight]"
         );
@@ -53,7 +52,7 @@ public static Iterable<Object[]> parameters() {
             suppliers,
             "fixed date with duration",
             () -> DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-02-17T09:00:00.00Z"),
-            EsqlDataTypes.TIME_DURATION,
+            DataTypes.TIME_DURATION,
             Duration.ofDays(1L),
             "[86400000 in Z][fixed]"
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
index dd7e36f23f5fa..ce7cd68198753 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
@@ -400,7 +400,7 @@ protected static void geoPoints(
         String evaluatorName,
         BiFunction<Integer, Stream<BytesRef>, Matcher<Object>> matcher
     ) {
-        geoPoints(cases, name, evaluatorName, EsqlDataTypes.GEO_POINT, matcher);
+        geoPoints(cases, name, evaluatorName, DataTypes.GEO_POINT, matcher);
     }
 
     /**
@@ -415,7 +415,7 @@ protected static void geoPoints(
         DataType expectedDataType,
         BiFunction<Integer, Stream<BytesRef>, Matcher<Object>> matcher
     ) {
-        spatial(cases, name, evaluatorName, EsqlDataTypes.GEO_POINT, expectedDataType, GEO, GeometryTestUtils::randomPoint, matcher);
+        spatial(cases, name, evaluatorName, DataTypes.GEO_POINT, expectedDataType, GEO, GeometryTestUtils::randomPoint, matcher);
     }
 
     /**
@@ -428,7 +428,7 @@ protected static void cartesianPoints(
         String evaluatorName,
         BiFunction<Integer, Stream<BytesRef>, Matcher<Object>> matcher
     ) {
-        cartesianPoints(cases, name, evaluatorName, EsqlDataTypes.CARTESIAN_POINT, matcher);
+        cartesianPoints(cases, name, evaluatorName, DataTypes.CARTESIAN_POINT, matcher);
     }
 
     /**
@@ -443,16 +443,7 @@ protected static void cartesianPoints(
         DataType expectedDataType,
         BiFunction<Integer, Stream<BytesRef>, Matcher<Object>> matcher
     ) {
-        spatial(
-            cases,
-            name,
-            evaluatorName,
-            EsqlDataTypes.CARTESIAN_POINT,
-            expectedDataType,
-            CARTESIAN,
-            ShapeTestUtils::randomPoint,
-            matcher
-        );
+        spatial(cases, name, evaluatorName, DataTypes.CARTESIAN_POINT, expectedDataType, CARTESIAN, ShapeTestUtils::randomPoint, matcher);
     }
 
     /**
@@ -471,7 +462,7 @@ protected static void geoShape(
             cases,
             name,
             evaluatorName,
-            EsqlDataTypes.GEO_SHAPE,
+            DataTypes.GEO_SHAPE,
             expectedDataType,
             GEO,
             () -> rarely() ? GeometryTestUtils.randomGeometry(randomBoolean()) : GeometryTestUtils.randomPoint(),
@@ -495,7 +486,7 @@ protected static void cartesianShape(
             cases,
             name,
             evaluatorName,
-            EsqlDataTypes.CARTESIAN_SHAPE,
+            DataTypes.CARTESIAN_SHAPE,
             expectedDataType,
             CARTESIAN,
             () -> rarely() ? ShapeTestUtils.randomGeometry(randomBoolean()) : ShapeTestUtils.randomPoint(),
@@ -630,7 +621,7 @@ private static <T extends Comparable<T>> void putInOrder(List<T> mvData, Block.M
 
     protected final DataType[] representableNumerics() {
         // TODO numeric should only include representable numbers but that is a change for a followup
-        return EsqlDataTypes.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new);
+        return DataTypes.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new);
     }
 
     protected DataType expectedType(List<DataType> argTypes) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java
index c5c22000671fc..3b4ab2b5cfaa0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java
@@ -33,11 +33,11 @@ public MvConcatTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testC
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType fieldType : EsqlDataTypes.types()) {
+        for (DataType fieldType : DataTypes.types()) {
             if (EsqlDataTypes.isString(fieldType) == false) {
                 continue;
             }
-            for (DataType delimType : EsqlDataTypes.types()) {
+            for (DataType delimType : DataTypes.types()) {
                 if (EsqlDataTypes.isString(delimType) == false) {
                     continue;
                 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java
index df2c959a3b66c..abd9a39f944d4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -39,10 +38,10 @@ public static Iterable<Object[]> parameters() {
         longs(cases, "mv_first", "MvFirst", DataTypes.LONG, (size, values) -> equalTo(values.findFirst().getAsLong()));
         unsignedLongs(cases, "mv_first", "MvFirst", DataTypes.UNSIGNED_LONG, (size, values) -> equalTo(values.findFirst().get()));
         dateTimes(cases, "mv_first", "MvFirst", DataTypes.DATETIME, (size, values) -> equalTo(values.findFirst().getAsLong()));
-        geoPoints(cases, "mv_first", "MvFirst", EsqlDataTypes.GEO_POINT, (size, values) -> equalTo(values.findFirst().get()));
-        cartesianPoints(cases, "mv_first", "MvFirst", EsqlDataTypes.CARTESIAN_POINT, (size, values) -> equalTo(values.findFirst().get()));
-        geoShape(cases, "mv_first", "MvFirst", EsqlDataTypes.GEO_SHAPE, (size, values) -> equalTo(values.findFirst().get()));
-        cartesianShape(cases, "mv_first", "MvFirst", EsqlDataTypes.CARTESIAN_SHAPE, (size, values) -> equalTo(values.findFirst().get()));
+        geoPoints(cases, "mv_first", "MvFirst", DataTypes.GEO_POINT, (size, values) -> equalTo(values.findFirst().get()));
+        cartesianPoints(cases, "mv_first", "MvFirst", DataTypes.CARTESIAN_POINT, (size, values) -> equalTo(values.findFirst().get()));
+        geoShape(cases, "mv_first", "MvFirst", DataTypes.GEO_SHAPE, (size, values) -> equalTo(values.findFirst().get()));
+        cartesianShape(cases, "mv_first", "MvFirst", DataTypes.CARTESIAN_SHAPE, (size, values) -> equalTo(values.findFirst().get()));
         return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(false, cases)));
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java
index d5616f90916ef..286b372dce584 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -39,22 +38,10 @@ public static Iterable<Object[]> parameters() {
         longs(cases, "mv_last", "MvLast", DataTypes.LONG, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsLong()));
         unsignedLongs(cases, "mv_last", "MvLast", DataTypes.UNSIGNED_LONG, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
         dateTimes(cases, "mv_last", "MvLast", DataTypes.DATETIME, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsLong()));
-        geoPoints(cases, "mv_last", "MvLast", EsqlDataTypes.GEO_POINT, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
-        cartesianPoints(
-            cases,
-            "mv_last",
-            "MvLast",
-            EsqlDataTypes.CARTESIAN_POINT,
-            (size, values) -> equalTo(values.reduce((f, s) -> s).get())
-        );
-        geoShape(cases, "mv_last", "MvLast", EsqlDataTypes.GEO_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
-        cartesianShape(
-            cases,
-            "mv_last",
-            "MvLast",
-            EsqlDataTypes.CARTESIAN_SHAPE,
-            (size, values) -> equalTo(values.reduce((f, s) -> s).get())
-        );
+        geoPoints(cases, "mv_last", "MvLast", DataTypes.GEO_POINT, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
+        cartesianPoints(cases, "mv_last", "MvLast", DataTypes.CARTESIAN_POINT, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
+        geoShape(cases, "mv_last", "MvLast", DataTypes.GEO_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
+        cartesianShape(cases, "mv_last", "MvLast", DataTypes.CARTESIAN_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
         return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(false, cases)));
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
index 184ac397f4e18..eba1bb623bb98 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -272,70 +271,70 @@ private static void bytesRefs(List<TestCaseSupplier> suppliers) {
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(EsqlDataTypes.GEO_POINT, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.GEO_POINT, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
             List<Object> field = randomList(1, 5, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint())));
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, EsqlDataTypes.GEO_POINT, "field"),
+                    new TestCaseSupplier.TypedData(field, DataTypes.GEO_POINT, "field"),
                     new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
                     new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                EsqlDataTypes.GEO_POINT,
+                DataTypes.GEO_POINT,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(EsqlDataTypes.CARTESIAN_POINT, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.CARTESIAN_POINT, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
             List<Object> field = randomList(1, 5, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint())));
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, EsqlDataTypes.CARTESIAN_POINT, "field"),
+                    new TestCaseSupplier.TypedData(field, DataTypes.CARTESIAN_POINT, "field"),
                     new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
                     new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                EsqlDataTypes.CARTESIAN_POINT,
+                DataTypes.CARTESIAN_POINT,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(EsqlDataTypes.GEO_SHAPE, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.GEO_SHAPE, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
             List<Object> field = randomList(1, 5, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))));
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, EsqlDataTypes.GEO_SHAPE, "field"),
+                    new TestCaseSupplier.TypedData(field, DataTypes.GEO_SHAPE, "field"),
                     new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
                     new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                EsqlDataTypes.GEO_SHAPE,
+                DataTypes.GEO_SHAPE,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(EsqlDataTypes.CARTESIAN_SHAPE, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.CARTESIAN_SHAPE, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
             List<Object> field = randomList(1, 5, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))));
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, EsqlDataTypes.CARTESIAN_SHAPE, "field"),
+                    new TestCaseSupplier.TypedData(field, DataTypes.CARTESIAN_SHAPE, "field"),
                     new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
                     new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                EsqlDataTypes.CARTESIAN_SHAPE,
+                DataTypes.CARTESIAN_SHAPE,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
index 7e925a1cb7f25..e884ba9400206 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunctionTestCase;
 import org.elasticsearch.xpack.esql.planner.Layout;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
@@ -91,12 +90,7 @@ public static Iterable<Object[]> parameters() {
     }
 
     protected static void addSpatialCombinations(List<TestCaseSupplier> suppliers) {
-        for (DataType dataType : List.of(
-            EsqlDataTypes.GEO_POINT,
-            EsqlDataTypes.GEO_SHAPE,
-            EsqlDataTypes.CARTESIAN_POINT,
-            EsqlDataTypes.CARTESIAN_SHAPE
-        )) {
+        for (DataType dataType : List.of(DataTypes.GEO_POINT, DataTypes.GEO_SHAPE, DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE)) {
             TestCaseSupplier.TypedDataSupplier leftDataSupplier = SpatialRelatesFunctionTestCase.testCaseSupplier(dataType);
             TestCaseSupplier.TypedDataSupplier rightDataSupplier = SpatialRelatesFunctionTestCase.testCaseSupplier(dataType);
             suppliers.add(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java
index 2b002e6c1e31f..b593ad4d3aed4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java
@@ -36,7 +36,7 @@ public IsNotNullTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> test
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType type : EsqlDataTypes.types()) {
+        for (DataType type : DataTypes.types()) {
             if (false == EsqlDataTypes.isRepresentable(type)) {
                 continue;
             }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java
index 15e6ee3672f1a..adeca03406ead 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java
@@ -36,7 +36,7 @@ public IsNullTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCas
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType type : EsqlDataTypes.types()) {
+        for (DataType type : DataTypes.types()) {
             if (false == EsqlDataTypes.isRepresentable(type)) {
                 continue;
             }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsTests.java
index 79edf8d53eca2..b29a952f4c792 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsTests.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -30,9 +30,9 @@ public SpatialContainsTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        DataType[] geoDataTypes = { EsqlDataTypes.GEO_POINT, EsqlDataTypes.GEO_SHAPE };
+        DataType[] geoDataTypes = { DataTypes.GEO_POINT, DataTypes.GEO_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, geoDataTypes);
-        DataType[] cartesianDataTypes = { EsqlDataTypes.CARTESIAN_POINT, EsqlDataTypes.CARTESIAN_SHAPE };
+        DataType[] cartesianDataTypes = { DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, cartesianDataTypes);
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), SpatialContainsTests::typeErrorMessage)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointTests.java
index f09975e6ff87b..999fc695fd8fe 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointTests.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -30,9 +30,9 @@ public SpatialDisjointTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        DataType[] geoDataTypes = { EsqlDataTypes.GEO_POINT, EsqlDataTypes.GEO_SHAPE };
+        DataType[] geoDataTypes = { DataTypes.GEO_POINT, DataTypes.GEO_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, geoDataTypes);
-        DataType[] cartesianDataTypes = { EsqlDataTypes.CARTESIAN_POINT, EsqlDataTypes.CARTESIAN_SHAPE };
+        DataType[] cartesianDataTypes = { DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, cartesianDataTypes);
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), SpatialDisjointTests::typeErrorMessage)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsTests.java
index 4a4d30046bfd2..19d43f42a9555 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsTests.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -30,9 +30,9 @@ public SpatialIntersectsTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCa
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        DataType[] geoDataTypes = { EsqlDataTypes.GEO_POINT, EsqlDataTypes.GEO_SHAPE };
+        DataType[] geoDataTypes = { DataTypes.GEO_POINT, DataTypes.GEO_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, geoDataTypes);
-        DataType[] cartesianDataTypes = { EsqlDataTypes.CARTESIAN_POINT, EsqlDataTypes.CARTESIAN_SHAPE };
+        DataType[] cartesianDataTypes = { DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, cartesianDataTypes);
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), SpatialIntersectsTests::typeErrorMessage)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinTests.java
index 9915d3d6ebeab..3dde6a1f83a3f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinTests.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -30,9 +30,9 @@ public SpatialWithinTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase>
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        DataType[] geoDataTypes = { EsqlDataTypes.GEO_POINT, EsqlDataTypes.GEO_SHAPE };
+        DataType[] geoDataTypes = { DataTypes.GEO_POINT, DataTypes.GEO_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, geoDataTypes);
-        DataType[] cartesianDataTypes = { EsqlDataTypes.CARTESIAN_POINT, EsqlDataTypes.CARTESIAN_SHAPE };
+        DataType[] cartesianDataTypes = { DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, cartesianDataTypes);
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), SpatialWithinTests::typeErrorMessage)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
index 9fc396daada99..9c8856280aa87 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
@@ -52,11 +52,11 @@ public static Iterable<Object[]> parameters() {
         }
         Set<DataType> supported = Set.of(DataTypes.NULL, DataTypes.KEYWORD, DataTypes.TEXT);
         List<Set<DataType>> supportedPerPosition = List.of(supported, supported);
-        for (DataType lhs : EsqlDataTypes.types()) {
+        for (DataType lhs : DataTypes.types()) {
             if (lhs == DataTypes.NULL || EsqlDataTypes.isRepresentable(lhs) == false) {
                 continue;
             }
-            for (DataType rhs : EsqlDataTypes.types()) {
+            for (DataType rhs : DataTypes.types()) {
                 if (rhs == DataTypes.NULL || EsqlDataTypes.isRepresentable(rhs) == false) {
                     continue;
                 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
index 84504d4cd5721..57883e0b84067 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
@@ -70,7 +70,7 @@ static Iterable<Object[]> parameters(Function<String, String> escapeString, Supp
         casesForString(cases, "3 bytes, 1 code point", () -> "☕", false, escapeString, optionalPattern);
         casesForString(cases, "6 bytes, 2 code points", () -> "❗️", false, escapeString, optionalPattern);
         casesForString(cases, "100 random code points", () -> randomUnicodeOfCodepointLength(100), true, escapeString, optionalPattern);
-        for (DataType type : EsqlDataTypes.types()) {
+        for (DataType type : DataTypes.types()) {
             if (type == DataTypes.KEYWORD || type == DataTypes.TEXT || type == DataTypes.NULL) {
                 continue;
             }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
index 97bf5fa21e164..f8cee3c2dd42f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -34,15 +33,15 @@ public ReplaceTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCa
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType strType : EsqlDataTypes.types()) {
+        for (DataType strType : DataTypes.types()) {
             if (DataTypes.isString(strType) == false) {
                 continue;
             }
-            for (DataType oldStrType : EsqlDataTypes.types()) {
+            for (DataType oldStrType : DataTypes.types()) {
                 if (DataTypes.isString(oldStrType) == false) {
                     continue;
                 }
-                for (DataType newStrType : EsqlDataTypes.types()) {
+                for (DataType newStrType : DataTypes.types()) {
                     if (DataTypes.isString(newStrType) == false) {
                         continue;
                     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java
index 93db2ab769551..b57183e9d56c2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.hamcrest.Matcher;
 
 import java.util.Arrays;
@@ -81,8 +80,8 @@ protected boolean supportsTypes(DataType lhsType, DataType rhsType) {
 
     public final void testApplyToAllTypes() {
         // TODO replace with test cases
-        for (DataType lhsType : EsqlDataTypes.types()) {
-            for (DataType rhsType : EsqlDataTypes.types()) {
+        for (DataType lhsType : DataTypes.types()) {
+            for (DataType rhsType : DataTypes.types()) {
                 if (supportsTypes(lhsType, rhsType) == false) {
                     continue;
                 }
@@ -118,12 +117,12 @@ public final void testApplyToAllTypes() {
     }
 
     public final void testResolveType() {
-        for (DataType lhsType : EsqlDataTypes.types()) {
+        for (DataType lhsType : DataTypes.types()) {
             if (isRepresentable(lhsType) == false) {
                 continue;
             }
             Literal lhs = randomLiteral(lhsType);
-            for (DataType rhsType : EsqlDataTypes.types()) {
+            for (DataType rhsType : DataTypes.types()) {
                 if (isRepresentable(rhsType) == false) {
                     continue;
                 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
index f61fb0a5b316d..4d6e6015073a9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.math.BigInteger;
 import java.time.Duration;
@@ -97,7 +96,7 @@ public static Iterable<Object[]> parameters() {
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 (lhs, rhs) -> ((Period) lhs).plus((Period) rhs),
-                EsqlDataTypes.DATE_PERIOD,
+                DataTypes.DATE_PERIOD,
                 TestCaseSupplier.datePeriodCases(),
                 TestCaseSupplier.datePeriodCases(),
                 startsWith("LiteralsEvaluator[lit="),  // lhs and rhs have to be literals, so we fold into a literal
@@ -108,7 +107,7 @@ public static Iterable<Object[]> parameters() {
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 (lhs, rhs) -> ((Duration) lhs).plus((Duration) rhs),
-                EsqlDataTypes.TIME_DURATION,
+                DataTypes.TIME_DURATION,
                 TestCaseSupplier.timeDurationCases(),
                 TestCaseSupplier.timeDurationCases(),
                 startsWith("LiteralsEvaluator[lit="), // lhs and rhs have to be literals, so we fold into a literal
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
index ecff55acff0c4..784599678d65b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.time.Duration;
 import java.time.Period;
@@ -95,20 +94,20 @@ public static Iterable<Object[]> parameters() {
         );
 
         // TODO: Wire up edge case generation functions for these
-        suppliers.addAll(List.of(new TestCaseSupplier("Duration", List.of(EsqlDataTypes.TIME_DURATION), () -> {
-            Duration arg = (Duration) randomLiteral(EsqlDataTypes.TIME_DURATION).value();
+        suppliers.addAll(List.of(new TestCaseSupplier("Duration", List.of(DataTypes.TIME_DURATION), () -> {
+            Duration arg = (Duration) randomLiteral(DataTypes.TIME_DURATION).value();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(arg, EsqlDataTypes.TIME_DURATION, "arg")),
+                List.of(new TestCaseSupplier.TypedData(arg, DataTypes.TIME_DURATION, "arg")),
                 "No evaluator since this expression is only folded",
-                EsqlDataTypes.TIME_DURATION,
+                DataTypes.TIME_DURATION,
                 equalTo(arg.negated())
             );
-        }), new TestCaseSupplier("Period", List.of(EsqlDataTypes.DATE_PERIOD), () -> {
-            Period arg = (Period) randomLiteral(EsqlDataTypes.DATE_PERIOD).value();
+        }), new TestCaseSupplier("Period", List.of(DataTypes.DATE_PERIOD), () -> {
+            Period arg = (Period) randomLiteral(DataTypes.DATE_PERIOD).value();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(arg, EsqlDataTypes.DATE_PERIOD, "arg")),
+                List.of(new TestCaseSupplier.TypedData(arg, DataTypes.DATE_PERIOD, "arg")),
                 "No evaluator since this expression is only folded",
-                EsqlDataTypes.DATE_PERIOD,
+                DataTypes.DATE_PERIOD,
                 equalTo(arg.negated())
             );
         })));
@@ -124,7 +123,7 @@ public void testEdgeCases() {
         // Run the assertions for the current test cases type only to avoid running the same assertions multiple times.
         // TODO: These remaining cases should get rolled into generation functions for periods and durations
         DataType testCaseType = testCase.getData().get(0).type();
-        if (testCaseType == EsqlDataTypes.DATE_PERIOD) {
+        if (testCaseType == DataTypes.DATE_PERIOD) {
             Period maxPeriod = Period.of(Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE);
             Period negatedMaxPeriod = Period.of(-Integer.MAX_VALUE, -Integer.MAX_VALUE, -Integer.MAX_VALUE);
             assertEquals(negatedMaxPeriod, process(maxPeriod));
@@ -136,7 +135,7 @@ public void testEdgeCases() {
                 () -> process(minPeriod)
             );
             assertEquals(e.getMessage(), "arithmetic exception in expression []: [integer overflow]");
-        } else if (testCaseType == EsqlDataTypes.TIME_DURATION) {
+        } else if (testCaseType == DataTypes.TIME_DURATION) {
             Duration maxDuration = Duration.ofSeconds(Long.MAX_VALUE, 0);
             Duration negatedMaxDuration = Duration.ofSeconds(-Long.MAX_VALUE, 0);
             assertEquals(negatedMaxDuration, process(maxDuration));
@@ -177,10 +176,10 @@ private static DataType typeOf(Object val) {
             return DataTypes.DOUBLE;
         }
         if (val instanceof Duration) {
-            return EsqlDataTypes.TIME_DURATION;
+            return DataTypes.TIME_DURATION;
         }
         if (val instanceof Period) {
-            return EsqlDataTypes.DATE_PERIOD;
+            return DataTypes.DATE_PERIOD;
         }
         throw new UnsupportedOperationException("unsupported type [" + val.getClass() + "]");
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
index e3e2af720d3e8..f3c63569d458d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.time.Duration;
 import java.time.Period;
@@ -82,11 +81,11 @@ public static Iterable<Object[]> parameters() {
 
         suppliers.add(new TestCaseSupplier("Datetime - Period", () -> {
             long lhs = (Long) randomLiteral(DataTypes.DATETIME).value();
-            Period rhs = (Period) randomLiteral(EsqlDataTypes.DATE_PERIOD).value();
+            Period rhs = (Period) randomLiteral(DataTypes.DATE_PERIOD).value();
             return new TestCaseSupplier.TestCase(
                 List.of(
                     new TestCaseSupplier.TypedData(lhs, DataTypes.DATETIME, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, EsqlDataTypes.DATE_PERIOD, "rhs")
+                    new TestCaseSupplier.TypedData(rhs, DataTypes.DATE_PERIOD, "rhs")
                 ),
                 "SubDatetimesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
                 DataTypes.DATETIME,
@@ -94,25 +93,25 @@ public static Iterable<Object[]> parameters() {
             );
         }));
         suppliers.add(new TestCaseSupplier("Period - Period", () -> {
-            Period lhs = (Period) randomLiteral(EsqlDataTypes.DATE_PERIOD).value();
-            Period rhs = (Period) randomLiteral(EsqlDataTypes.DATE_PERIOD).value();
+            Period lhs = (Period) randomLiteral(DataTypes.DATE_PERIOD).value();
+            Period rhs = (Period) randomLiteral(DataTypes.DATE_PERIOD).value();
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(lhs, EsqlDataTypes.DATE_PERIOD, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, EsqlDataTypes.DATE_PERIOD, "rhs")
+                    new TestCaseSupplier.TypedData(lhs, DataTypes.DATE_PERIOD, "lhs"),
+                    new TestCaseSupplier.TypedData(rhs, DataTypes.DATE_PERIOD, "rhs")
                 ),
                 "Only folding possible, so there's no evaluator",
-                EsqlDataTypes.DATE_PERIOD,
+                DataTypes.DATE_PERIOD,
                 equalTo(lhs.minus(rhs))
             );
         }));
         suppliers.add(new TestCaseSupplier("Datetime - Duration", () -> {
             long lhs = (Long) randomLiteral(DataTypes.DATETIME).value();
-            Duration rhs = (Duration) randomLiteral(EsqlDataTypes.TIME_DURATION).value();
+            Duration rhs = (Duration) randomLiteral(DataTypes.TIME_DURATION).value();
             TestCaseSupplier.TestCase testCase = new TestCaseSupplier.TestCase(
                 List.of(
                     new TestCaseSupplier.TypedData(lhs, DataTypes.DATETIME, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, EsqlDataTypes.TIME_DURATION, "rhs")
+                    new TestCaseSupplier.TypedData(rhs, DataTypes.TIME_DURATION, "rhs")
                 ),
                 "SubDatetimesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
                 DataTypes.DATETIME,
@@ -121,15 +120,15 @@ public static Iterable<Object[]> parameters() {
             return testCase;
         }));
         suppliers.add(new TestCaseSupplier("Duration - Duration", () -> {
-            Duration lhs = (Duration) randomLiteral(EsqlDataTypes.TIME_DURATION).value();
-            Duration rhs = (Duration) randomLiteral(EsqlDataTypes.TIME_DURATION).value();
+            Duration lhs = (Duration) randomLiteral(DataTypes.TIME_DURATION).value();
+            Duration rhs = (Duration) randomLiteral(DataTypes.TIME_DURATION).value();
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(lhs, EsqlDataTypes.TIME_DURATION, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, EsqlDataTypes.TIME_DURATION, "rhs")
+                    new TestCaseSupplier.TypedData(lhs, DataTypes.TIME_DURATION, "lhs"),
+                    new TestCaseSupplier.TypedData(rhs, DataTypes.TIME_DURATION, "rhs")
                 ),
                 "Only folding possible, so there's no evaluator",
-                EsqlDataTypes.TIME_DURATION,
+                DataTypes.TIME_DURATION,
                 equalTo(lhs.minus(rhs))
             );
         }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
index 7d147c03dbb75..feaee83476302 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
@@ -101,7 +101,6 @@
 import org.elasticsearch.xpack.esql.plan.physical.RowExec;
 import org.elasticsearch.xpack.esql.plan.physical.ShowExec;
 import org.elasticsearch.xpack.esql.plan.physical.TopNExec;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -690,7 +689,7 @@ static Map<String, EsField> randomProperties(int depth) {
         return Map.copyOf(map);
     }
 
-    static List<DataType> DATA_TYPES = EsqlDataTypes.types().stream().toList();
+    static List<DataType> DATA_TYPES = DataTypes.types().stream().toList();
 
     static DataType randomDataType() {
         return DATA_TYPES.get(randomIntBetween(0, DATA_TYPES.size() - 1));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
index 86c53ae839d63..cfe2535519557 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
@@ -152,6 +152,8 @@
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
@@ -164,8 +166,6 @@
 import static org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison.BinaryComparisonOperation.GTE;
 import static org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison.BinaryComparisonOperation.LT;
 import static org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison.BinaryComparisonOperation.LTE;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.contains;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
index 7a37d61bed089..a57754d594af9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
@@ -127,12 +127,12 @@
 import static org.elasticsearch.xpack.esql.core.expression.Expressions.names;
 import static org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.ASC;
 import static org.elasticsearch.xpack.esql.core.expression.function.scalar.FunctionTestUtils.l;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
 import static org.elasticsearch.xpack.esql.parser.ExpressionBuilder.MAX_EXPRESSION_DEPTH;
 import static org.elasticsearch.xpack.esql.parser.LogicalPlanBuilder.MAX_QUERY_DEPTH;
 import static org.elasticsearch.xpack.esql.plan.physical.AggregateExec.Mode.FINAL;
 import static org.elasticsearch.xpack.esql.plan.physical.AggregateExec.Mode.PARTIAL;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -257,7 +257,7 @@ private static EnrichResolution setupEnrichResolution() {
                     Map.entry("city", new EsField("city", DataTypes.KEYWORD, Map.of(), true)),
                     Map.entry("airport", new EsField("airport", DataTypes.TEXT, Map.of(), false)),
                     Map.entry("region", new EsField("region", DataTypes.TEXT, Map.of(), false)),
-                    Map.entry("city_boundary", new EsField("city_boundary", EsqlDataTypes.GEO_SHAPE, Map.of(), false))
+                    Map.entry("city_boundary", new EsField("city_boundary", DataTypes.GEO_SHAPE, Map.of(), false))
                 )
             )
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
index 8853f4661fe9b..09dae4c739115 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
@@ -44,12 +44,12 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
 import static org.elasticsearch.xpack.esql.core.expression.function.FunctionResolutionStrategy.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.TIME_DURATION;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
index d85f1c0964de3..d14537aede31e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
@@ -51,7 +51,6 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.time.Duration;
 import java.time.ZoneOffset;
@@ -84,7 +83,7 @@ public class EvalMapperTests extends ESTestCase {
     public static List<Object[]> params() {
         Literal literal = new Literal(Source.EMPTY, new BytesRef("something"), DataTypes.KEYWORD);
         Literal datePattern = new Literal(Source.EMPTY, new BytesRef("yyyy"), DataTypes.KEYWORD);
-        Literal dateInterval = new Literal(Source.EMPTY, Duration.ofHours(1), EsqlDataTypes.TIME_DURATION);
+        Literal dateInterval = new Literal(Source.EMPTY, Duration.ofHours(1), DataTypes.TIME_DURATION);
 
         List<Object[]> params = new ArrayList<>();
         for (Expression e : new Expression[] {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
index b72bd5c5d6fb5..8134dc16aedf6 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeSubclassTests;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow;
@@ -35,7 +36,6 @@
 import org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType;
 import org.elasticsearch.xpack.esql.plan.physical.OutputExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.lang.reflect.Modifier;
@@ -158,7 +158,7 @@ static String randomGrokPattern() {
         );
     }
 
-    static List<DataType> DATA_TYPES = EsqlDataTypes.types().stream().toList();
+    static List<DataType> DATA_TYPES = DataTypes.types().stream().toList();
 
     static EsQueryExec.FieldSort randomFieldSort() {
         return new EsQueryExec.FieldSort(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java
index 61b5f1e5fdd51..9eaef21421f55 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java
@@ -25,9 +25,9 @@
 public class EsqlDataTypeRegistryTests extends ESTestCase {
 
     public void testCounter() {
-        resolve("long", TimeSeriesParams.MetricType.COUNTER, EsqlDataTypes.COUNTER_LONG);
-        resolve("integer", TimeSeriesParams.MetricType.COUNTER, EsqlDataTypes.COUNTER_INTEGER);
-        resolve("double", TimeSeriesParams.MetricType.COUNTER, EsqlDataTypes.COUNTER_DOUBLE);
+        resolve("long", TimeSeriesParams.MetricType.COUNTER, DataTypes.COUNTER_LONG);
+        resolve("integer", TimeSeriesParams.MetricType.COUNTER, DataTypes.COUNTER_INTEGER);
+        resolve("double", TimeSeriesParams.MetricType.COUNTER, DataTypes.COUNTER_DOUBLE);
 
     }
 

From b142a8013fa4d2341ff65b3469ae6b4a36b3e501 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 May 2024 12:46:03 -0400
Subject: [PATCH 044/208] ESQL: Fix ESQL lookup tests in release build
 (#109175)

The release build gets a different error message.

Closes #109170
---
 .../org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index b770c9b93f4d5..5f2b4290f48f3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.analysis;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -1883,6 +1884,10 @@ public void testLookup() {
             | RENAME languages AS int
             | LOOKUP int_number_names ON int
             """));
+        if (Build.current().isProductionRelease()) {
+            assertThat(e.getMessage(), containsString("line 3:4: LOOKUP is in preview and only available in SNAPSHOT build"));
+            return;
+        }
         assertThat(e.getMessage(), containsString("LOOKUP not yet supported"));
     }
 

From ceb941f48cc833144b0717d17874f4d7ba5910f8 Mon Sep 17 00:00:00 2001
From: George Wallace <georgewallace@users.noreply.github.com>
Date: Wed, 29 May 2024 11:57:46 -0600
Subject: [PATCH 045/208] Removing server from the version compatibility table
 (#109168)

* Removing server from the version compatibility table

* fixed typo

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../sql/endpoints/version-compat.asciidoc     | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/docs/reference/sql/endpoints/version-compat.asciidoc b/docs/reference/sql/endpoints/version-compat.asciidoc
index 27cff939d2174..dd9d0f51eae4f 100644
--- a/docs/reference/sql/endpoints/version-compat.asciidoc
+++ b/docs/reference/sql/endpoints/version-compat.asciidoc
@@ -1,11 +1,11 @@
-Your driver must be compatible with your {es} server version.
+Your driver must be compatible with your {es} version.
 
-IMPORTANT: The driver version cannot be newer than the {es} server version.
-For example, A 7.10.0 server is not compatible with {version} drivers.
+IMPORTANT: The driver version cannot be newer than the {es} version.
+For example, {es} version 7.10.0 is not compatible with {version} drivers.
 
 [options="header",cols="1,3a,1"]
 |====
-| {es} server version
+| {es} version
 | Compatible driver versions
 | Example
 
@@ -16,22 +16,21 @@ ifeval::[ "{minor-version}" != "8.0" ]
 | * The same version
   * Any earlier 8.x version
   * Any 7.x version after 7.7.0.
-| An {version} server is compatible with {version} and earlier 8.x drivers. An
-{version} server is also compatible with 7.7.0 and later 7.x drivers.
+| {es} {version} is compatible with {version} and earlier 8.x drivers. {es} {version} is also compatible with 7.7.0 and later 7.x drivers.
 endif::[]
 
 ifeval::[ "{minor-version}" == "8.0" ]
 | 8.0.0
 | * The same version
   * Any 7.x version after 7.7.0.
-| An 8.0.0 server is compatible with 8.0.0 drivers. An 8.0.0 server is also
+| {es} 8.0.0 is compatible with 8.0.0 drivers. {es} 8.0.0 is also
 compatible with 7.7.0 and later 7.x drivers.
 endif::[]
 
 | 7.7.1-{prev-major-last}
 | * The same version
   * An earlier 7.x version, back to 7.7.0.
-| A 7.10.0 server is compatible with 7.7.0-7.10.0 drivers.
+| {es} 7.10.0 is compatible with 7.7.0-7.10.0 drivers.
 
 endif::[]
 
@@ -39,10 +38,10 @@ ifeval::[ "{major-version}" == "7.x" ]
 | 7.7.1-{version}
 | * The same version
   * An earlier 7.x version, back to 7.7.0.
-| A 7.10.0 server is compatible with 7.7.0-7.10.0 drivers.
+| {es} 7.10.0 is compatible with 7.7.0-7.10.0 drivers.
 endif::[]
 
 | 7.7.0 and earlier versions
 | * The same version.
-| A 7.6.1 server is only compatible with 7.6.1 drivers.
+| {es} 7.6.1 is only compatible with 7.6.1 drivers.
 |====

From f21f50065abef7cd91fc446163d23cbf52a9d4cd Mon Sep 17 00:00:00 2001
From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com>
Date: Wed, 29 May 2024 14:57:52 -0400
Subject: [PATCH 046/208] [ML] Ignoring duplicate keys for _cat trained models
 and usage (#109126)

* Handling duplicate keys

* Update docs/changelog/109126.yaml

* adding tests

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 docs/changelog/109126.yaml                    |  5 ++
 ...TrainedModelsStatsActionResponseTests.java | 22 +++--
 .../MachineLearningUsageTransportAction.java  | 16 +++-
 .../rest/cat/RestCatTrainedModelsAction.java  | 64 ++++++++++----
 ...hineLearningUsageTransportActionTests.java | 57 ++++++++++++
 .../cat/RestCatTrainedModelsActionTests.java  | 86 +++++++++++++++++++
 6 files changed, 224 insertions(+), 26 deletions(-)
 create mode 100644 docs/changelog/109126.yaml
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportActionTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java

diff --git a/docs/changelog/109126.yaml b/docs/changelog/109126.yaml
new file mode 100644
index 0000000000000..248eacc76b65c
--- /dev/null
+++ b/docs/changelog/109126.yaml
@@ -0,0 +1,5 @@
+pr: 109126
+summary: Correctly handle duplicate model ids for the `_cat` trained models api and usage statistics
+area: Machine Learning
+type: bug
+issues: [ ]
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java
index 4aebe3369ce48..8c175c17fccc8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java
@@ -30,6 +30,15 @@ public class GetTrainedModelsStatsActionResponseTests extends AbstractBWCWireSer
 
     @Override
     protected Response createTestInstance() {
+        return createInstance();
+    }
+
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
+    public static Response createInstance() {
         int listSize = randomInt(10);
         List<Response.TrainedModelStats> trainedModelStats = Stream.generate(() -> randomAlphaOfLength(10))
             .limit(listSize)
@@ -47,12 +56,7 @@ protected Response createTestInstance() {
         return new Response(new QueryPage<>(trainedModelStats, randomLongBetween(listSize, 1000), RESULTS_FIELD));
     }
 
-    @Override
-    protected Response mutateInstance(Response instance) {
-        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
-    }
-
-    private IngestStats randomIngestStats() {
+    public static IngestStats randomIngestStats() {
         List<String> pipelineIds = Stream.generate(() -> randomAlphaOfLength(10)).limit(randomIntBetween(0, 10)).toList();
         return new IngestStats(
             new IngestStats.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()),
@@ -61,15 +65,15 @@ private IngestStats randomIngestStats() {
         );
     }
 
-    private IngestStats.Stats randomStats() {
+    private static IngestStats.Stats randomStats() {
         return new IngestStats.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
     }
 
-    private IngestStats.ByteStats randomByteStats() {
+    private static IngestStats.ByteStats randomByteStats() {
         return new IngestStats.ByteStats(randomNonNegativeLong(), randomNonNegativeLong());
     }
 
-    private List<IngestStats.ProcessorStat> randomProcessorStats() {
+    private static List<IngestStats.ProcessorStat> randomProcessorStats() {
         return Stream.generate(() -> randomAlphaOfLength(10))
             .limit(randomIntBetween(0, 10))
             .map(name -> new IngestStats.ProcessorStat(name, "inference", randomStats()))
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
index d24d9b13f065d..e5540ef7e8c5f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
@@ -63,6 +63,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.TreeMap;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -503,7 +504,8 @@ private static void addDeploymentStats(
         );
     }
 
-    private static void addTrainedModelStats(
+    // Default for testing
+    static void addTrainedModelStats(
         GetTrainedModelsAction.Response modelsResponse,
         GetTrainedModelsStatsAction.Response statsResponse,
         Map<String, Object> inferenceUsage
@@ -512,7 +514,17 @@ private static void addTrainedModelStats(
         Map<String, GetTrainedModelsStatsAction.Response.TrainedModelStats> statsToModelId = statsResponse.getResources()
             .results()
             .stream()
-            .collect(Collectors.toMap(GetTrainedModelsStatsAction.Response.TrainedModelStats::getModelId, Function.identity()));
+            .filter(Objects::nonNull)
+            .collect(
+                Collectors.toMap(
+                    GetTrainedModelsStatsAction.Response.TrainedModelStats::getModelId,
+                    Function.identity(),
+                    // Addresses issue: https://github.com/elastic/elasticsearch/issues/108423
+                    // There could be multiple deployments of the same model which would result in a collision, since all we need is the
+                    // memory used by the model we can use either one
+                    (stats1, stats2) -> stats1
+                )
+            );
         Map<String, Object> trainedModelsUsage = new HashMap<>();
         trainedModelsUsage.put(MachineLearningFeatureSetUsage.ALL, createCountUsageEntry(trainedModelConfigs.size()));
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java
index f53a82f54a67e..d4a2b602759ad 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java
@@ -14,7 +14,8 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.ingest.IngestStats;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -37,6 +38,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -245,19 +247,52 @@ protected Table getTableWithHeader(RestRequest request) {
         return table;
     }
 
-    private Table buildTable(
+    private record AccumulatedStats(int pipelineCount, IngestStats ingestStats) {
+        public static final AccumulatedStats EMPTY = of(null);
+
+        public static AccumulatedStats of(@Nullable GetTrainedModelsStatsAction.Response.TrainedModelStats stats) {
+            return new AccumulatedStats(stats == null ? 0 : stats.getPipelineCount(), getIngestStats(stats));
+        }
+
+        private static IngestStats getIngestStats(@Nullable GetTrainedModelsStatsAction.Response.TrainedModelStats stats) {
+            if (stats == null) {
+                return IngestStats.IDENTITY;
+            }
+
+            return stats.getIngestStats() == null ? IngestStats.IDENTITY : stats.getIngestStats();
+        }
+
+        public static AccumulatedStats merge(AccumulatedStats first, AccumulatedStats second) {
+            return new AccumulatedStats(
+                first.pipelineCount + second.pipelineCount,
+                IngestStats.merge(first.ingestStats, second.ingestStats)
+            );
+        }
+    }
+
+    // Default for testing
+    Table buildTable(
         RestRequest request,
         List<GetTrainedModelsStatsAction.Response.TrainedModelStats> stats,
         List<TrainedModelConfig> configs,
         List<DataFrameAnalyticsConfig> analyticsConfigs
     ) {
         Table table = getTableWithHeader(request);
-        assert configs.size() == stats.size();
 
         Map<String, DataFrameAnalyticsConfig> analyticsMap = analyticsConfigs.stream()
             .collect(Collectors.toMap(DataFrameAnalyticsConfig::getId, Function.identity()));
-        Map<String, GetTrainedModelsStatsAction.Response.TrainedModelStats> statsMap = stats.stream()
-            .collect(Collectors.toMap(GetTrainedModelsStatsAction.Response.TrainedModelStats::getModelId, Function.identity()));
+        Map<String, AccumulatedStats> accumulatedStatsMap = stats.stream()
+            .filter(Objects::nonNull)
+            .collect(
+                Collectors.toMap(
+                    GetTrainedModelsStatsAction.Response.TrainedModelStats::getModelId,
+                    AccumulatedStats::of,
+                    // If there are multiple deployments of the same model we'll need to total the stats
+                    AccumulatedStats::merge
+                )
+            );
+
+        assert configs.size() == accumulatedStatsMap.size();
 
         configs.forEach(config -> {
             table.startRow();
@@ -272,17 +307,16 @@ private Table buildTable(
             table.addCell(config.getDescription());
             table.addCell(config.getModelType());
 
-            GetTrainedModelsStatsAction.Response.TrainedModelStats modelStats = statsMap.get(config.getModelId());
-            table.addCell(modelStats.getPipelineCount());
-            boolean hasIngestStats = modelStats != null && modelStats.getIngestStats() != null;
-            table.addCell(hasIngestStats ? modelStats.getIngestStats().totalStats().ingestCount() : 0);
-            table.addCell(
-                hasIngestStats
-                    ? TimeValue.timeValueMillis(modelStats.getIngestStats().totalStats().ingestTimeInMillis())
-                    : TimeValue.timeValueMillis(0)
+            AccumulatedStats accumulatedStats = Objects.requireNonNullElse(
+                accumulatedStatsMap.get(config.getModelId()),
+                AccumulatedStats.EMPTY
             );
-            table.addCell(hasIngestStats ? modelStats.getIngestStats().totalStats().ingestCurrent() : 0);
-            table.addCell(hasIngestStats ? modelStats.getIngestStats().totalStats().ingestFailedCount() : 0);
+
+            table.addCell(accumulatedStats.pipelineCount);
+            table.addCell(accumulatedStats.ingestStats.totalStats().ingestCount());
+            table.addCell(accumulatedStats.ingestStats.totalStats().ingestTimeInMillis());
+            table.addCell(accumulatedStats.ingestStats.totalStats().ingestCurrent());
+            table.addCell(accumulatedStats.ingestStats.totalStats().ingestFailedCount());
 
             DataFrameAnalyticsConfig dataFrameAnalyticsConfig = config.getTags()
                 .stream()
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportActionTests.java
new file mode 100644
index 0000000000000..579d30b3e46c4
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportActionTests.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ml;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.action.util.QueryPage;
+import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
+import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsAction;
+import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsActionResponseTests;
+import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
+import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfigTests;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModelSizeStatsTests;
+import org.elasticsearch.xpack.core.ml.stats.StatsAccumulator;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+public class MachineLearningUsageTransportActionTests extends ESTestCase {
+    public void testAddTrainedModelStatsHandlesMultipleDeployments() {
+        Map<String, Object> usage = new HashMap<>();
+
+        var deploymentConfig = TrainedModelConfigTests.createTestInstance("id1").build();
+        var stats = new GetTrainedModelsStatsAction.Response.TrainedModelStats(
+            "id1",
+            TrainedModelSizeStatsTests.createRandom(),
+            GetTrainedModelsStatsActionResponseTests.randomIngestStats(),
+            randomIntBetween(0, 10),
+            null,
+            null
+        );
+        StatsAccumulator actualMemoryUsage = new StatsAccumulator();
+        actualMemoryUsage.add(stats.getModelSizeStats().getModelSizeBytes());
+
+        var modelsResponse = new GetTrainedModelsAction.Response(
+            new QueryPage<>(List.of(deploymentConfig), 1, GetTrainedModelsAction.Response.RESULTS_FIELD)
+        );
+
+        var statsResponse = new GetTrainedModelsStatsAction.Response(
+            new QueryPage<>(List.of(stats), 1, GetTrainedModelsStatsAction.Response.RESULTS_FIELD)
+        );
+
+        MachineLearningUsageTransportAction.addTrainedModelStats(modelsResponse, statsResponse, usage);
+        @SuppressWarnings("unchecked")
+        var expectedModelMemoryUsage = ((Map<String, Object>) usage.get("trained_models")).get(
+            TrainedModelConfig.MODEL_SIZE_BYTES.getPreferredName()
+        );
+        assertThat(expectedModelMemoryUsage, is(actualMemoryUsage.asMap()));
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java
new file mode 100644
index 0000000000000..f7cfba048e722
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ml.rest.cat;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsAction;
+import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsActionResponseTests;
+import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfigTests;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModelSizeStats;
+import org.junit.Before;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+
+public class RestCatTrainedModelsActionTests extends ESTestCase {
+
+    private RestCatTrainedModelsAction action;
+
+    @Before
+    public void setUpAction() {
+        action = new RestCatTrainedModelsAction();
+    }
+
+    public void testBuildTableAccumulatedStats() {
+        // GetTrainedModelsStatsActionResponseTests
+        var deployment1 = new GetTrainedModelsStatsAction.Response.TrainedModelStats(
+            "id1",
+            new TrainedModelSizeStats(100, 200),
+            GetTrainedModelsStatsActionResponseTests.randomIngestStats(),
+            2,
+            null,
+            null
+        );
+
+        var deployment2 = new GetTrainedModelsStatsAction.Response.TrainedModelStats(
+            "id1",
+            new TrainedModelSizeStats(1, 2),
+            GetTrainedModelsStatsActionResponseTests.randomIngestStats(),
+            2,
+            null,
+            null
+        );
+
+        var configs = List.of(TrainedModelConfigTests.createTestInstance("id1").build());
+
+        var table = action.buildTable(new FakeRestRequest(), List.of(deployment1, deployment2), configs, List.of());
+        assertThat(table.getRows().get(0).get(0).value, is("id1"));
+        // pipeline count
+        assertThat(table.getRows().get(0).get(9).value, is(4));
+        // ingest count
+        assertThat(
+            table.getRows().get(0).get(10).value,
+            is(deployment1.getIngestStats().totalStats().ingestCount() + deployment2.getIngestStats().totalStats().ingestCount())
+        );
+        // ingest time in millis
+        assertThat(
+            table.getRows().get(0).get(11).value,
+            is(
+                deployment1.getIngestStats().totalStats().ingestTimeInMillis() + deployment2.getIngestStats()
+                    .totalStats()
+                    .ingestTimeInMillis()
+            )
+        );
+        // ingest current
+        assertThat(
+            table.getRows().get(0).get(12).value,
+            is(deployment1.getIngestStats().totalStats().ingestCurrent() + deployment2.getIngestStats().totalStats().ingestCurrent())
+        );
+        // ingest failed count
+        assertThat(
+            table.getRows().get(0).get(13).value,
+            is(
+                deployment1.getIngestStats().totalStats().ingestFailedCount() + deployment2.getIngestStats()
+                    .totalStats()
+                    .ingestFailedCount()
+            )
+        );
+    }
+}

From 1bd1071979d38f2b86ad5b3b0248b01672f7a6cd Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 May 2024 15:20:52 -0400
Subject: [PATCH 047/208] ESQL: Make `NameId` `Writeable` (#109164)

This exposes the read and write logic for `NameId` to ESQL-core so we
can implement `Writeable` on more things to make ESQL more like the rest
of Elasticsearch. Reading a `NameId` has to go through the resolution
logic so you still need to read it via the `PlanStreamInput`. `NameId`
is just special.
---
 .../xpack/esql/core/expression/NameId.java    | 20 +++++++++-
 .../xpack/esql/core/util/PlanStreamInput.java | 10 +++++
 .../xpack/esql/io/stream/PlanNamedTypes.java  | 38 +++++++++----------
 .../xpack/esql/io/stream/PlanStreamInput.java | 14 +++----
 .../esql/io/stream/PlanNamedTypesTests.java   |  6 +--
 5 files changed, 56 insertions(+), 32 deletions(-)

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java
index d643c6fedcb70..2aa70397075d0 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java
@@ -6,6 +6,12 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
+
+import java.io.IOException;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -16,9 +22,9 @@
  * and that create reproducible values when run in subsequent
  * tests. They don't produce reproducible values in production, but
  * you rarely debug with them in production and commonly do so in
- * tests.
+ * tests.</p>
  */
-public class NameId {
+public class NameId implements Writeable {
     private static final AtomicLong COUNTER = new AtomicLong();
     private final long id;
 
@@ -47,4 +53,14 @@ public boolean equals(Object obj) {
     public String toString() {
         return Long.toString(id);
     }
+
+    public static <S extends StreamInput & PlanStreamInput> NameId readFrom(S in) throws IOException {
+        long unmappedId = in.readLong();
+        return in.mapNameId(unmappedId);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeLong(id);
+    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
index 51898f36d5497..df8fac06dd478 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
@@ -7,8 +7,11 @@
 
 package org.elasticsearch.xpack.esql.core.util;
 
+import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
+import java.io.IOException;
+
 /**
  * Interface for streams that can serialize plan components. This exists so
  * ESQL proper can expose streaming capability to ESQL-core. If the world is kind
@@ -23,4 +26,11 @@ public interface PlanStreamInput {
      */
     String sourceText();
 
+    /**
+     * Translate a {@code long} into a {@link NameId}, mapping the same {@code long}
+     * into the same {@link NameId} each time. Each new {@code long} gets assigned
+     * a unique id to the node, but when the same id is sent in the stream we get
+     * the same result.
+     */
+    NameId mapNameId(long id) throws IOException;
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index 09a514e2c14f7..33c5937c0fd0a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
+import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.Order;
@@ -224,7 +225,6 @@
 import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.Entry.of;
 import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader;
 import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter;
-import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToLong;
 
 /**
  * A utility class that consists solely of static methods that describe how to serialize and
@@ -1074,21 +1074,21 @@ static FieldAttribute readFieldAttribute(PlanStreamInput in) throws IOException
             in.readEsFieldNamed(),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
-            in.nameIdFromLongValue(in.readLong()),
+            NameId.readFrom(in),
             in.readBoolean()
         );
     }
 
-    static void writeFieldAttribute(PlanStreamOutput out, FieldAttribute fileAttribute) throws IOException {
+    static void writeFieldAttribute(PlanStreamOutput out, FieldAttribute fieldAttribute) throws IOException {
         Source.EMPTY.writeTo(out);
-        out.writeOptionalWriteable(fileAttribute.parent() == null ? null : o -> writeFieldAttribute(out, fileAttribute.parent()));
-        out.writeString(fileAttribute.name());
-        out.writeString(fileAttribute.dataType().typeName());
-        out.writeNamed(EsField.class, fileAttribute.field());
-        out.writeOptionalString(fileAttribute.qualifier());
-        out.writeEnum(fileAttribute.nullable());
-        out.writeLong(stringToLong(fileAttribute.id().toString()));
-        out.writeBoolean(fileAttribute.synthetic());
+        out.writeOptionalWriteable(fieldAttribute.parent() == null ? null : o -> writeFieldAttribute(out, fieldAttribute.parent()));
+        out.writeString(fieldAttribute.name());
+        out.writeString(fieldAttribute.dataType().typeName());
+        out.writeNamed(EsField.class, fieldAttribute.field());
+        out.writeOptionalString(fieldAttribute.qualifier());
+        out.writeEnum(fieldAttribute.nullable());
+        fieldAttribute.id().writeTo(out);
+        out.writeBoolean(fieldAttribute.synthetic());
     }
 
     static ReferenceAttribute readReferenceAttr(PlanStreamInput in) throws IOException {
@@ -1098,7 +1098,7 @@ static ReferenceAttribute readReferenceAttr(PlanStreamInput in) throws IOExcepti
             in.dataTypeFromTypeName(in.readString()),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
-            in.nameIdFromLongValue(in.readLong()),
+            NameId.readFrom(in),
             in.readBoolean()
         );
     }
@@ -1109,7 +1109,7 @@ static void writeReferenceAttr(PlanStreamOutput out, ReferenceAttribute referenc
         out.writeString(referenceAttribute.dataType().typeName());
         out.writeOptionalString(referenceAttribute.qualifier());
         out.writeEnum(referenceAttribute.nullable());
-        out.writeLong(stringToLong(referenceAttribute.id().toString()));
+        referenceAttribute.id().writeTo(out);
         out.writeBoolean(referenceAttribute.synthetic());
     }
 
@@ -1120,7 +1120,7 @@ static MetadataAttribute readMetadataAttr(PlanStreamInput in) throws IOException
             in.dataTypeFromTypeName(in.readString()),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
-            in.nameIdFromLongValue(in.readLong()),
+            NameId.readFrom(in),
             in.readBoolean(),
             in.readBoolean()
         );
@@ -1132,7 +1132,7 @@ static void writeMetadataAttr(PlanStreamOutput out, MetadataAttribute metadataAt
         out.writeString(metadataAttribute.dataType().typeName());
         out.writeOptionalString(metadataAttribute.qualifier());
         out.writeEnum(metadataAttribute.nullable());
-        out.writeLong(stringToLong(metadataAttribute.id().toString()));
+        metadataAttribute.id().writeTo(out);
         out.writeBoolean(metadataAttribute.synthetic());
         out.writeBoolean(metadataAttribute.searchable());
     }
@@ -1143,7 +1143,7 @@ static UnsupportedAttribute readUnsupportedAttr(PlanStreamInput in) throws IOExc
             in.readString(),
             readUnsupportedEsField(in),
             in.readOptionalString(),
-            in.nameIdFromLongValue(in.readLong())
+            NameId.readFrom(in)
         );
     }
 
@@ -1152,7 +1152,7 @@ static void writeUnsupportedAttr(PlanStreamOutput out, UnsupportedAttribute unsu
         out.writeString(unsupportedAttribute.name());
         writeUnsupportedEsField(out, unsupportedAttribute.field());
         out.writeOptionalString(unsupportedAttribute.hasCustomMessage() ? unsupportedAttribute.unresolvedMessage() : null);
-        out.writeLong(stringToLong(unsupportedAttribute.id().toString()));
+        unsupportedAttribute.id().writeTo(out);
     }
 
     // -- EsFields
@@ -1856,7 +1856,7 @@ static Alias readAlias(PlanStreamInput in) throws IOException {
             in.readString(),
             in.readOptionalString(),
             in.readNamed(Expression.class),
-            in.nameIdFromLongValue(in.readLong()),
+            NameId.readFrom(in),
             in.readBoolean()
         );
     }
@@ -1866,7 +1866,7 @@ static void writeAlias(PlanStreamOutput out, Alias alias) throws IOException {
         out.writeString(alias.name());
         out.writeOptionalString(alias.qualifier());
         out.writeExpression(alias.child());
-        out.writeLong(stringToLong(alias.id().toString()));
+        alias.id().writeTo(out);
         out.writeBoolean(alias.synthetic());
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
index c8dc24e4c2208..d5b1990e69c5f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
@@ -45,7 +45,6 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.function.LongFunction;
-import java.util.function.Supplier;
 
 /**
  * A customized stream input used to deserialize ESQL physical plan fragments. Complements stream
@@ -70,8 +69,6 @@ public NameId apply(long streamNameId) {
         }
     }
 
-    private static final Supplier<LongFunction<NameId>> DEFAULT_NAME_ID_FUNC = NameIdMapper::new;
-
     private final Map<Integer, Block> cachedBlocks = new HashMap<>();
 
     private final PlanNameRegistry registry;
@@ -90,11 +87,7 @@ public PlanStreamInput(
         super(streamInput, namedWriteableRegistry);
         this.registry = registry;
         this.configuration = configuration;
-        this.nameIdFunction = DEFAULT_NAME_ID_FUNC.get();
-    }
-
-    NameId nameIdFromLongValue(long value) {
-        return nameIdFunction.apply(value);
+        this.nameIdFunction = new NameIdMapper();
     }
 
     DataType dataTypeFromTypeName(String typeName) throws IOException {
@@ -278,4 +271,9 @@ static void throwOnNullOptionalRead(PlanReader<?> reader) throws IOException {
         assert false : e;
         throw e;
     }
+
+    @Override
+    public NameId mapNameId(long l) {
+        return nameIdFunction.apply(l);
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
index feaee83476302..e05abefba66a7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
@@ -234,7 +234,7 @@ public void testUnsupportedAttributeSimple() throws IOException {
         var in = planStreamInput(bso);
         var deser = PlanNamedTypes.readUnsupportedAttr(in);
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-        assertThat(deser.id(), equalTo(in.nameIdFromLongValue(Long.parseLong(orig.id().toString()))));
+        assertThat(deser.id(), equalTo(in.mapNameId(Long.parseLong(orig.id().toString()))));
     }
 
     public void testUnsupportedAttribute() {
@@ -259,7 +259,7 @@ public void testFieldAttributeSimple() throws IOException {
         var in = planStreamInput(bso);
         var deser = PlanNamedTypes.readFieldAttribute(in);
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-        assertThat(deser.id(), equalTo(in.nameIdFromLongValue(Long.parseLong(orig.id().toString()))));
+        assertThat(deser.id(), equalTo(in.mapNameId(Long.parseLong(orig.id().toString()))));
     }
 
     public void testFieldAttribute() {
@@ -413,7 +413,7 @@ public void testAliasSimple() throws IOException {
         var in = planStreamInput(bso);
         var deser = PlanNamedTypes.readAlias(in);
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-        assertThat(deser.id(), equalTo(in.nameIdFromLongValue(Long.parseLong(orig.id().toString()))));
+        assertThat(deser.id(), equalTo(in.mapNameId(Long.parseLong(orig.id().toString()))));
     }
 
     public void testLiteralSimple() throws IOException {

From f420aa4ffa0df295c4e1dda04ae7bc5eb2ebfed3 Mon Sep 17 00:00:00 2001
From: Artem Prigoda <arteamon@gmail.com>
Date: Wed, 29 May 2024 23:41:35 +0200
Subject: [PATCH 048/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109188

---
 muted-tests.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index 9d78a6714e35e..ef5fe6a190767 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -39,6 +39,10 @@ tests:
   issue: "https://github.com/elastic/elasticsearch/issues/109154"
   method: "test {p0=data_stream/200_rollover_failure_store/Lazily roll over a data\
     \ stream's failure store after an ingest failure}"
+- class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109188"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
+    \ dependent mappings / dates}"
 # Examples:
 #
 #  Mute a single test case in a YAML test suite:

From 3309850016e1e983da9a2ca1a0e7fdf7fa675bd8 Mon Sep 17 00:00:00 2001
From: Artem Prigoda <arteamon@gmail.com>
Date: Wed, 29 May 2024 23:42:49 +0200
Subject: [PATCH 049/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109189

---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index ef5fe6a190767..08862ebf99fe9 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -43,6 +43,9 @@ tests:
   issue: "https://github.com/elastic/elasticsearch/issues/109188"
   method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
     \ dependent mappings / dates}"
+- class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109189"
+  method: "test {p0=esql/70_locale/Date format with Italian locale}"
 # Examples:
 #
 #  Mute a single test case in a YAML test suite:

From b8894c39ed000ff2d02ae6a59a7045f0e1d0e60b Mon Sep 17 00:00:00 2001
From: Artem Prigoda <arteamon@gmail.com>
Date: Wed, 29 May 2024 23:53:21 +0200
Subject: [PATCH 050/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109190

---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index 08862ebf99fe9..212f3cd02fe4a 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -46,6 +46,9 @@ tests:
 - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109189"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
+- class: "org.elasticsearch.upgrades.SearchStatesIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109190"
+  method: "testBWCSearchStates"
 # Examples:
 #
 #  Mute a single test case in a YAML test suite:

From db1c76bef514fa36935e14f51a9430a528f8c94e Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 29 May 2024 15:30:02 -0700
Subject: [PATCH 051/208] Guard against a null scorer in painless execute
 (#109048)

The painless execute api allows for a sample document and query to be
provided. If the query does not actually match the document, a null
scorer is produced. This commit guards against that condition, returning
a clearer error message indicating what happened.

closes #43541
---
 docs/changelog/109048.yaml                                  | 6 ++++++
 .../painless/action/PainlessExecuteAction.java              | 3 +++
 2 files changed, 9 insertions(+)
 create mode 100644 docs/changelog/109048.yaml

diff --git a/docs/changelog/109048.yaml b/docs/changelog/109048.yaml
new file mode 100644
index 0000000000000..8bae082404ecd
--- /dev/null
+++ b/docs/changelog/109048.yaml
@@ -0,0 +1,6 @@
+pr: 109048
+summary: Guard against a null scorer in painless execute
+area: Infra/Scripting
+type: bug
+issues:
+ - 43541
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
index 6ab5fc724c711..0736fd4ef4a87 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
@@ -648,6 +648,9 @@ static Response innerShardOperation(Request request, ScriptService scriptService
                         luceneQuery = indexSearcher.rewrite(luceneQuery);
                         Weight weight = indexSearcher.createWeight(luceneQuery, ScoreMode.COMPLETE, 1f);
                         Scorer scorer = weight.scorer(indexSearcher.getIndexReader().leaves().get(0));
+                        if (scorer == null) {
+                            throw new IllegalArgumentException("The provided query did not match the sample document");
+                        }
                         // Consume the first (and only) match.
                         int docID = scorer.iterator().nextDoc();
                         assert docID == scorer.docID();

From e022bc365c68e0dafb4c22234dcba4e765adb2d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20Fred=C3=A9n?=
 <109296772+jfreden@users.noreply.github.com>
Date: Thu, 30 May 2024 09:12:52 +0200
Subject: [PATCH 052/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109198

---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index 212f3cd02fe4a..a4267a9565d71 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -49,6 +49,9 @@ tests:
 - class: "org.elasticsearch.upgrades.SearchStatesIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109190"
   method: "testBWCSearchStates"
+- class: "org.elasticsearch.xpack.esql.CsvTests"
+  issue: "https://github.com/elastic/elasticsearch/issues/109198"
+  method: "test {ip.IpPrefixLengthFromColumn}"
 # Examples:
 #
 #  Mute a single test case in a YAML test suite:

From fe7e70ee6bdda65301a95b45b900b5225af7e86c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20Fred=C3=A9n?=
 <109296772+jfreden@users.noreply.github.com>
Date: Thu, 30 May 2024 09:41:51 +0200
Subject: [PATCH 053/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109200

---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index a4267a9565d71..262b1509ca58e 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -52,6 +52,9 @@ tests:
 - class: "org.elasticsearch.xpack.esql.CsvTests"
   issue: "https://github.com/elastic/elasticsearch/issues/109198"
   method: "test {ip.IpPrefixLengthFromColumn}"
+- class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109200"
+  method: "test {p0=esql/70_locale/Date format with Italian locale}"
 # Examples:
 #
 #  Mute a single test case in a YAML test suite:

From 060ccc53b944466fd3872e42cc35dba02c2d4fbc Mon Sep 17 00:00:00 2001
From: Albert Zaharovits <albert.zaharovits@elastic.co>
Date: Thu, 30 May 2024 11:35:59 +0300
Subject: [PATCH 054/208] Fixes cluster state-based role mappings not recovered
 from disk (#109167)

Cluster state role mappings need to be recovered from disk CS
(when master nodes are restarted), because the CS metadata
that says if the role mappings are up to date (it actually tells
if all file settings are up to date) is itself recovered from disk.
In case that the metadata is recovered, but the role mappings
themselves are not, the role mappings from the file will not
be present in the CS and will also not be copied from the
settings file.

Fixes: #93048
---
 docs/changelog/109167.yaml                    |   5 +
 .../service/FileSettingsServiceIT.java        |  23 ++-
 .../xpack/core/XPackClientPlugin.java         |   6 +
 .../security/authz/RoleMappingMetadata.java   |  27 +++
 .../RoleMappingFileSettingsIT.java            |   4 +-
 .../FileSettingsRoleMappingsRestartIT.java    | 182 +++++++++++-------
 ...ingMetadataXContentSerializationTests.java |  77 ++++++++
 .../mapper/ExpressionRoleMappingTests.java    |   2 +-
 8 files changed, 239 insertions(+), 87 deletions(-)
 create mode 100644 docs/changelog/109167.yaml
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RoleMappingMetadataXContentSerializationTests.java

diff --git a/docs/changelog/109167.yaml b/docs/changelog/109167.yaml
new file mode 100644
index 0000000000000..e366b2302263c
--- /dev/null
+++ b/docs/changelog/109167.yaml
@@ -0,0 +1,5 @@
+pr: 109167
+summary: Fixes cluster state-based role mappings not recovered from disk
+area: Authorization
+type: bug
+issues: []
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java
index 38bc372868df0..6e89c1447edb6 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.reservedstate.service;
 
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
@@ -103,7 +104,7 @@ private void assertMasterNode(Client client, String node) {
         assertThat(client.admin().cluster().prepareState().get().getState().nodes().getMasterNode().getName(), equalTo(node));
     }
 
-    private void writeJSONFile(String node, String json) throws Exception {
+    public static void writeJSONFile(String node, String json, AtomicLong versionCounter, Logger logger) throws Exception {
         long version = versionCounter.incrementAndGet();
 
         FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
@@ -111,9 +112,11 @@ private void writeJSONFile(String node, String json) throws Exception {
         Files.createDirectories(fileSettingsService.watchedFileDir());
         Path tempFilePath = createTempFile();
 
-        Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8));
+        String settingsFileContent = Strings.format(json, version);
+        Files.write(tempFilePath, settingsFileContent.getBytes(StandardCharsets.UTF_8));
+        logger.info("--> Before writing new settings file with version [{}]", version);
         Files.move(tempFilePath, fileSettingsService.watchedFile(), StandardCopyOption.ATOMIC_MOVE);
-        logger.info("--> New file settings: [{}]", Strings.format(json, version));
+        logger.info("--> After writing new settings file: [{}]", settingsFileContent);
     }
 
     private Tuple<CountDownLatch, AtomicLong> setupCleanupClusterStateListener(String node) {
@@ -203,7 +206,7 @@ public void testSettingsApplied() throws Exception {
         assertTrue(masterFileSettingsService.watching());
         assertFalse(dataFileSettingsService.watching());
 
-        writeJSONFile(masterNode, testJSON);
+        writeJSONFile(masterNode, testJSON, versionCounter, logger);
         assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
     }
 
@@ -218,7 +221,7 @@ public void testSettingsAppliedOnStart() throws Exception {
 
         // In internal cluster tests, the nodes share the config directory, so when we write with the data node path
         // the master will pick it up on start
-        writeJSONFile(dataNode, testJSON);
+        writeJSONFile(dataNode, testJSON, versionCounter, logger);
 
         logger.info("--> start master node");
         final String masterNode = internalCluster().startMasterOnlyNode();
@@ -246,7 +249,7 @@ public void testReservedStatePersistsOnRestart() throws Exception {
         assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
 
         logger.info("--> write some settings");
-        writeJSONFile(masterNode, testJSON);
+        writeJSONFile(masterNode, testJSON, versionCounter, logger);
         assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
 
         logger.info("--> restart master");
@@ -325,7 +328,7 @@ public void testErrorSaved() throws Exception {
         assertTrue(masterFileSettingsService.watching());
         assertFalse(dataFileSettingsService.watching());
 
-        writeJSONFile(masterNode, testErrorJSON);
+        writeJSONFile(masterNode, testErrorJSON, versionCounter, logger);
         assertClusterStateNotSaved(savedClusterState.v1(), savedClusterState.v2());
     }
 
@@ -349,7 +352,7 @@ public void testSettingsAppliedOnMasterReElection() throws Exception {
 
         assertTrue(masterFileSettingsService.watching());
 
-        writeJSONFile(masterNode, testJSON);
+        writeJSONFile(masterNode, testJSON, versionCounter, logger);
         assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
 
         internalCluster().stopCurrentMasterNode();
@@ -364,13 +367,13 @@ public void testSettingsAppliedOnMasterReElection() throws Exception {
         ensureStableCluster(3);
 
         savedClusterState = setupCleanupClusterStateListener(internalCluster().getMasterName());
-        writeJSONFile(internalCluster().getMasterName(), testCleanupJSON);
+        writeJSONFile(internalCluster().getMasterName(), testCleanupJSON, versionCounter, logger);
 
         boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
         assertTrue(awaitSuccessful);
 
         savedClusterState = setupClusterStateListener(internalCluster().getMasterName());
-        writeJSONFile(internalCluster().getMasterName(), testJSON43mb);
+        writeJSONFile(internalCluster().getMasterName(), testJSON43mb, versionCounter, logger);
 
         assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "43mb");
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index 9f6d502435d3a..a2c3e40c76ae4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -376,10 +376,16 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
                 new ParseField(TransformMetadata.TYPE),
                 parser -> TransformMetadata.LENIENT_PARSER.parse(parser, null).build()
             ),
+            // security
             new NamedXContentRegistry.Entry(
                 PersistentTaskParams.class,
                 new ParseField(SecurityMigrationTaskParams.TASK_NAME),
                 SecurityMigrationTaskParams::fromXContent
+            ),
+            new NamedXContentRegistry.Entry(
+                Metadata.Custom.class,
+                new ParseField(RoleMappingMetadata.TYPE),
+                RoleMappingMetadata::fromXContent
             )
         );
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java
index cc523ed55c095..da6ff6ad24c34 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java
@@ -17,21 +17,43 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.EnumSet;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.Objects;
 import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.Metadata.ALL_CONTEXTS;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 
 public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Custom> implements Metadata.Custom {
 
     public static final String TYPE = "role_mappings";
 
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<RoleMappingMetadata, Void> PARSER = new ConstructingObjectParser<>(
+        TYPE,
+        // serialization tests rely on the order of the ExpressionRoleMapping
+        args -> new RoleMappingMetadata(new LinkedHashSet<>((Collection<ExpressionRoleMapping>) args[0]))
+    );
+
+    static {
+        PARSER.declareObjectArray(
+            constructorArg(),
+            // role mapping names are lost when the role mapping metadata is serialized
+            (p, c) -> ExpressionRoleMapping.parse("name_not_available_after_deserialization", p),
+            new ParseField(TYPE)
+        );
+    }
+
     private static final RoleMappingMetadata EMPTY = new RoleMappingMetadata(Set.of());
 
     public static RoleMappingMetadata getFromClusterState(ClusterState clusterState) {
@@ -71,9 +93,14 @@ public static NamedDiff<Metadata.Custom> readDiffFrom(StreamInput streamInput) t
 
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        // role mappings are serialized without their names
         return Iterators.concat(ChunkedToXContentHelper.startArray(TYPE), roleMappings.iterator(), ChunkedToXContentHelper.endArray());
     }
 
+    public static RoleMappingMetadata fromXContent(XContentParser parser) throws IOException {
+        return PARSER.apply(parser, null);
+    }
+
     @Override
     public String getWriteableName() {
         return TYPE;
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
index 286a9cb736b1b..21a5b53e89af7 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
@@ -180,7 +180,7 @@ public static void writeJSONFile(String node, String json, Logger logger, Atomic
         logger.info("--> after writing JSON config to node {} with path {}", node, tempFilePath);
     }
 
-    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node, String expectedKey) {
+    public static Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node, String expectedKey) {
         ClusterService clusterService = internalCluster().clusterService(node);
         CountDownLatch savedClusterState = new CountDownLatch(1);
         AtomicLong metadataVersion = new AtomicLong(-1);
@@ -202,7 +202,7 @@ public void clusterChanged(ClusterChangedEvent event) {
         return new Tuple<>(savedClusterState, metadataVersion);
     }
 
-    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForCleanup(String node) {
+    public static Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForCleanup(String node) {
         ClusterService clusterService = internalCluster().clusterService(node);
         CountDownLatch savedClusterState = new CountDownLatch(1);
         AtomicLong metadataVersion = new AtomicLong(-1);
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java
index 6e4e5fcc2a631..032cab446df2a 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java
@@ -8,33 +8,26 @@
 package org.elasticsearch.xpack.security;
 
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterStateListener;
-import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
-import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.core.Strings;
-import org.elasticsearch.core.Tuple;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.reservedstate.service.FileSettingsService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.SecurityIntegTestCase;
-import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction;
-import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
+import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
 import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction;
 
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.util.Arrays;
-import java.util.concurrent.CountDownLatch;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Collectors;
 
-import static org.hamcrest.Matchers.allOf;
+import static org.elasticsearch.integration.RoleMappingFileSettingsIT.setupClusterStateListener;
+import static org.elasticsearch.integration.RoleMappingFileSettingsIT.setupClusterStateListenerForCleanup;
+import static org.elasticsearch.integration.RoleMappingFileSettingsIT.writeJSONFile;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.emptyIterable;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class FileSettingsRoleMappingsRestartIT extends SecurityIntegTestCase {
@@ -59,7 +52,7 @@ public class FileSettingsRoleMappingsRestartIT extends SecurityIntegTestCase {
                           }
                        },
                        "everyone_fleet_alone": {
-                          "enabled": true,
+                          "enabled": false,
                           "roles": [ "fleet_user" ],
                           "rules": { "field": { "username": "*" } },
                           "metadata": {
@@ -71,84 +64,125 @@ public class FileSettingsRoleMappingsRestartIT extends SecurityIntegTestCase {
              }
         }""";
 
-    private void writeJSONFile(String node, String json) throws Exception {
-        long version = versionCounter.incrementAndGet();
-
-        FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
-
-        Files.deleteIfExists(fileSettingsService.watchedFile());
-
-        Files.createDirectories(fileSettingsService.watchedFileDir());
-        Path tempFilePath = createTempFile();
-
-        logger.info("--> writing JSON config to node {} with path {}", node, tempFilePath);
-        logger.info(Strings.format(json, version));
-        Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8));
-        Files.move(tempFilePath, fileSettingsService.watchedFile(), StandardCopyOption.ATOMIC_MOVE);
-    }
-
-    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node, String expectedKey) {
-        ClusterService clusterService = internalCluster().clusterService(node);
-        CountDownLatch savedClusterState = new CountDownLatch(1);
-        AtomicLong metadataVersion = new AtomicLong(-1);
-        clusterService.addListener(new ClusterStateListener() {
-            @Override
-            public void clusterChanged(ClusterChangedEvent event) {
-                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
-                if (reservedState != null) {
-                    ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedRoleMappingAction.NAME);
-                    if (handlerMetadata != null && handlerMetadata.keys().contains(expectedKey)) {
-                        clusterService.removeListener(this);
-                        metadataVersion.set(event.state().metadata().version());
-                        savedClusterState.countDown();
-                    }
-                }
-            }
-        });
-
-        return new Tuple<>(savedClusterState, metadataVersion);
-    }
+    private static String emptyJSON = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                "cluster_settings": {},
+                "role_mappings": {}
+             }
+        }""";
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/93048")
     public void testReservedStatePersistsOnRestart() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(0);
 
         final String masterNode = internalCluster().getMasterName();
         var savedClusterState = setupClusterStateListener(masterNode, "everyone_kibana_alone");
 
-        FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
-
-        assertTrue(masterFileSettingsService.watching());
-
+        awaitFileSettingsWatcher();
         logger.info("--> write some role mappings, no other file settings");
-        writeJSONFile(masterNode, testJSONOnlyRoleMappings);
+        writeJSONFile(masterNode, testJSONOnlyRoleMappings, logger, versionCounter);
         boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
         assertTrue(awaitSuccessful);
 
+        var clusterState = clusterAdmin().state(new ClusterStateRequest()).actionGet().getState();
+        assertRoleMappingReservedMetadata(clusterState, "everyone_kibana_alone", "everyone_fleet_alone");
+        List<ExpressionRoleMapping> roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings());
+        assertThat(
+            roleMappings,
+            containsInAnyOrder(
+                new ExpressionRoleMapping(
+                    "everyone_kibana_alone",
+                    new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                    List.of("kibana_user"),
+                    List.of(),
+                    Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something"),
+                    true
+                ),
+                new ExpressionRoleMapping(
+                    "everyone_fleet_alone",
+                    new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                    List.of("fleet_user"),
+                    List.of(),
+                    Map.of("uuid", "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7", "_foo", "something_else"),
+                    false
+                )
+            )
+        );
+
         logger.info("--> restart master");
         internalCluster().restartNode(masterNode);
+        ensureGreen();
+
+        // assert role mappings are recovered from "disk"
+        clusterState = clusterAdmin().state(new ClusterStateRequest()).actionGet().getState();
+        assertRoleMappingReservedMetadata(clusterState, "everyone_kibana_alone", "everyone_fleet_alone");
+        roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings());
+        assertThat(
+            roleMappings,
+            containsInAnyOrder(
+                new ExpressionRoleMapping(
+                    "name_not_available_after_deserialization",
+                    new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                    List.of("kibana_user"),
+                    List.of(),
+                    Map.of("uuid", "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7", "_foo", "something"),
+                    true
+                ),
+                new ExpressionRoleMapping(
+                    "name_not_available_after_deserialization",
+                    new FieldExpression("username", List.of(new FieldExpression.FieldValue("*"))),
+                    List.of("fleet_user"),
+                    List.of(),
+                    Map.of("uuid", "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7", "_foo", "something_else"),
+                    false
+                )
+            )
+        );
+
+        // now remove the role mappings via the same settings file
+        savedClusterState = setupClusterStateListenerForCleanup(masterNode);
+        awaitFileSettingsWatcher();
+        logger.info("--> remove the role mappings with an empty settings file");
+        writeJSONFile(masterNode, emptyJSON, logger, versionCounter);
+        awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
 
+        clusterState = clusterAdmin().state(new ClusterStateRequest()).actionGet().getState();
+        assertRoleMappingReservedMetadata(clusterState);
+        roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings());
+        assertThat(roleMappings, emptyIterable());
+
+        // and restart the master to confirm the role mappings are all gone
+        logger.info("--> restart master again");
+        internalCluster().restartNode(masterNode);
         ensureGreen();
 
-        var clusterStateResponse = clusterAdmin().state(new ClusterStateRequest()).actionGet();
+        // assert empty role mappings are recovered from "disk"
+        clusterState = clusterAdmin().state(new ClusterStateRequest()).actionGet().getState();
+        assertRoleMappingReservedMetadata(clusterState);
+        roleMappings = new ArrayList<>(RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings());
+        assertThat(roleMappings, emptyIterable());
+    }
+
+    private void assertRoleMappingReservedMetadata(ClusterState clusterState, String... names) {
         assertThat(
-            clusterStateResponse.getState()
-                .metadata()
+            clusterState.metadata()
                 .reservedStateMetadata()
                 .get(FileSettingsService.NAMESPACE)
                 .handlers()
                 .get(ReservedRoleMappingAction.NAME)
                 .keys(),
-            containsInAnyOrder("everyone_fleet_alone", "everyone_kibana_alone")
+            containsInAnyOrder(names)
         );
+    }
 
-        var request = new GetRoleMappingsRequest();
-        request.setNames("everyone_kibana_alone", "everyone_fleet_alone");
-        var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
-        assertTrue(response.hasMappings());
-        assertThat(
-            Arrays.stream(response.mappings()).map(r -> r.getName()).collect(Collectors.toSet()),
-            allOf(notNullValue(), containsInAnyOrder("everyone_kibana_alone", "everyone_fleet_alone"))
-        );
+    private void awaitFileSettingsWatcher() throws Exception {
+        final String masterNode = internalCluster().getMasterName();
+        FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+        assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RoleMappingMetadataXContentSerializationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RoleMappingMetadataXContentSerializationTests.java
new file mode 100644
index 0000000000000..db0201a1c072b
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RoleMappingMetadataXContentSerializationTests.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.XPackClientPlugin;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.security.authc.support.mapper.ExpressionRoleMappingTests.randomRoleMapping;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RoleMappingMetadataXContentSerializationTests extends AbstractChunkedSerializingTestCase<RoleMappingMetadata> {
+    @Override
+    protected Writeable.Reader<RoleMappingMetadata> instanceReader() {
+        return RoleMappingMetadata::new;
+    }
+
+    @Override
+    protected RoleMappingMetadata createTestInstance() {
+        return new RoleMappingMetadata(randomSet(0, 3, () -> randomRoleMapping(true)));
+    }
+
+    @Override
+    protected RoleMappingMetadata mutateInstance(RoleMappingMetadata instance) throws IOException {
+        Set<ExpressionRoleMapping> mutatedRoleMappings = new HashSet<>(instance.getRoleMappings());
+        boolean mutated = false;
+        if (mutatedRoleMappings.isEmpty() == false && randomBoolean()) {
+            mutated = true;
+            mutatedRoleMappings.remove(randomFrom(mutatedRoleMappings));
+        }
+        if (randomBoolean() || mutated == false) {
+            mutatedRoleMappings.add(randomRoleMapping(true));
+        }
+        return new RoleMappingMetadata(mutatedRoleMappings);
+    }
+
+    @Override
+    protected RoleMappingMetadata doParseInstance(XContentParser parser) throws IOException {
+        return RoleMappingMetadata.fromXContent(parser);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(new XPackClientPlugin().getNamedWriteables());
+    }
+
+    @Override
+    protected void assertEqualInstances(RoleMappingMetadata expectedInstance, RoleMappingMetadata newInstance) {
+        assertThat(expectedInstance.getRoleMappings().size(), equalTo(newInstance.getRoleMappings().size()));
+        for (ExpressionRoleMapping expectedExpressionRoleMapping : expectedInstance.getRoleMappings()) {
+            boolean found = false;
+            for (ExpressionRoleMapping newExpressionRoleMapping : newInstance.getRoleMappings()) {
+                // everything equals except name, because the name is lost during deserialization of {@code RoleMappingMetadata}
+                found |= newExpressionRoleMapping.isEnabled() == expectedExpressionRoleMapping.isEnabled()
+                    && Objects.equals(newExpressionRoleMapping.getExpression(), expectedExpressionRoleMapping.getExpression())
+                    && Objects.equals(newExpressionRoleMapping.getRoles(), expectedExpressionRoleMapping.getRoles())
+                    && Objects.equals(newExpressionRoleMapping.getRoleTemplates(), expectedExpressionRoleMapping.getRoleTemplates())
+                    && Objects.equals(newExpressionRoleMapping.getMetadata(), expectedExpressionRoleMapping.getMetadata());
+            }
+            assertTrue(found);
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExpressionRoleMappingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExpressionRoleMappingTests.java
index cfb85a133719f..fc5eb135343b9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExpressionRoleMappingTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExpressionRoleMappingTests.java
@@ -484,7 +484,7 @@ public static ExpressionRoleMapping randomRoleMapping(boolean acceptRoleTemplate
             randomAlphaOfLengthBetween(3, 8),
             new FieldExpression(
                 randomAlphaOfLengthBetween(4, 12),
-                Collections.singletonList(new FieldExpression.FieldValue(randomInt(99)))
+                Collections.singletonList(new FieldExpression.FieldValue((long) randomInt(99)))
             ),
             roles,
             templates,

From 0ae817e5b60d3801cb02a2b7586573dd8ca72b1d Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Thu, 30 May 2024 10:49:26 +0200
Subject: [PATCH 055/208] Complete sparse file tracker gaps before releasing
 refs on CacheFileRegion (#107359)

This change ensure that CacheFileRegion has references at the time
the Gap is completed. It also ensure that a reference on CacheFileRegion
is held until all writes/gaps are completed (ie not acquiring a ref before
writing each gap anymore).

Finally, it ensure that the Gap is correctly failed if the write task is rejected.
---
 .../shared/SharedBlobCacheService.java        | 146 ++++++++----------
 1 file changed, 66 insertions(+), 80 deletions(-)

diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
index 218f12a97e1a6..c3cd60d88c6f2 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.RefCountingListener;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.action.support.UnsafePlainActionFuture;
 import org.elasticsearch.blobcache.BlobCacheMetrics;
 import org.elasticsearch.blobcache.BlobCacheUtils;
@@ -31,7 +32,6 @@
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.Assertions;
 import org.elasticsearch.core.Releasable;
-import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
@@ -908,29 +908,30 @@ void populate(
             final Executor executor,
             final ActionListener<Boolean> listener
         ) {
-            Releasable resource = null;
             try {
                 incRefEnsureOpen();
-                resource = Releasables.releaseOnce(this::decRef);
-                final List<SparseFileTracker.Gap> gaps = tracker.waitForRange(
-                    rangeToWrite,
-                    rangeToWrite,
-                    Assertions.ENABLED ? ActionListener.releaseAfter(ActionListener.running(() -> {
-                        assert regionOwners.get(io) == this;
-                    }), resource) : ActionListener.releasing(resource)
-                );
-                final var hasGapsToFill = gaps.size() > 0;
-                try (RefCountingListener refs = new RefCountingListener(listener.map(unused -> hasGapsToFill))) {
-                    if (hasGapsToFill) {
-                        final var cacheFileRegion = CacheFileRegion.this;
+                try (RefCountingRunnable refs = new RefCountingRunnable(CacheFileRegion.this::decRef)) {
+                    final List<SparseFileTracker.Gap> gaps = tracker.waitForRange(
+                        rangeToWrite,
+                        rangeToWrite,
+                        Assertions.ENABLED ? ActionListener.releaseAfter(ActionListener.running(() -> {
+                            assert regionOwners.get(io) == this;
+                        }), refs.acquire()) : refs.acquireListener()
+                    );
+                    if (gaps.isEmpty()) {
+                        listener.onResponse(false);
+                        return;
+                    }
+                    try (var gapsListener = new RefCountingListener(listener.map(unused -> true))) {
                         for (SparseFileTracker.Gap gap : gaps) {
-                            var fillGapRunnable = fillGapRunnable(cacheFileRegion, writer, gap);
-                            executor.execute(ActionRunnable.run(refs.acquire(), fillGapRunnable::run));
+                            executor.execute(
+                                fillGapRunnable(gap, writer, ActionListener.releaseAfter(gapsListener.acquire(), refs.acquire()))
+                            );
                         }
                     }
                 }
             } catch (Exception e) {
-                releaseAndFail(listener, resource, e);
+                listener.onFailure(e);
             }
         }
 
@@ -942,77 +943,62 @@ void populateAndRead(
             final Executor executor,
             final ActionListener<Integer> listener
         ) {
-            Releasable resource = null;
             try {
                 incRefEnsureOpen();
-                resource = Releasables.releaseOnce(this::decRef);
-                final List<SparseFileTracker.Gap> gaps = tracker.waitForRange(
-                    rangeToWrite,
-                    rangeToRead,
-                    ActionListener.runAfter(listener, resource::close).delegateFailureAndWrap((l, success) -> {
-                        var ioRef = io;
-                        assert regionOwners.get(ioRef) == this;
-                        final int start = Math.toIntExact(rangeToRead.start());
-                        final int read = reader.onRangeAvailable(ioRef, start, start, Math.toIntExact(rangeToRead.length()));
-                        assert read == rangeToRead.length()
-                            : "partial read ["
-                                + read
-                                + "] does not match the range to read ["
-                                + rangeToRead.end()
-                                + '-'
-                                + rangeToRead.start()
-                                + ']';
-                        readCount.increment();
-                        l.onResponse(read);
-                    })
-                );
-
-                if (gaps.isEmpty() == false) {
-                    final var cacheFileRegion = CacheFileRegion.this;
-                    for (SparseFileTracker.Gap gap : gaps) {
-                        executor.execute(fillGapRunnable(cacheFileRegion, writer, gap));
+                try (RefCountingRunnable refs = new RefCountingRunnable(CacheFileRegion.this::decRef)) {
+                    final List<SparseFileTracker.Gap> gaps = tracker.waitForRange(
+                        rangeToWrite,
+                        rangeToRead,
+                        ActionListener.releaseAfter(listener, refs.acquire()).delegateFailureAndWrap((l, success) -> {
+                            var ioRef = io;
+                            assert regionOwners.get(ioRef) == this;
+                            final int start = Math.toIntExact(rangeToRead.start());
+                            final int read = reader.onRangeAvailable(ioRef, start, start, Math.toIntExact(rangeToRead.length()));
+                            assert read == rangeToRead.length()
+                                : "partial read ["
+                                    + read
+                                    + "] does not match the range to read ["
+                                    + rangeToRead.end()
+                                    + '-'
+                                    + rangeToRead.start()
+                                    + ']';
+                            readCount.increment();
+                            l.onResponse(read);
+                        })
+                    );
+
+                    if (gaps.isEmpty() == false) {
+                        for (SparseFileTracker.Gap gap : gaps) {
+                            executor.execute(fillGapRunnable(gap, writer, refs.acquireListener()));
+                        }
                     }
                 }
             } catch (Exception e) {
-                releaseAndFail(listener, resource, e);
-            }
-        }
-
-        private AbstractRunnable fillGapRunnable(CacheFileRegion cacheFileRegion, RangeMissingHandler writer, SparseFileTracker.Gap gap) {
-            return new AbstractRunnable() {
-                @Override
-                protected void doRun() throws Exception {
-                    if (cacheFileRegion.tryIncRefEnsureOpen() == false) {
-                        throw new AlreadyClosedException("File chunk [" + cacheFileRegion.regionKey + "] has been released");
-                    }
-                    try {
-                        final int start = Math.toIntExact(gap.start());
-                        var ioRef = io;
-                        assert regionOwners.get(ioRef) == cacheFileRegion;
-                        writer.fillCacheRange(
-                            ioRef,
-                            start,
-                            start,
-                            Math.toIntExact(gap.end() - start),
-                            progress -> gap.onProgress(start + progress)
-                        );
-                        writeCount.increment();
-                    } finally {
-                        cacheFileRegion.decRef();
-                    }
-                    gap.onCompletion();
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    gap.onFailure(e);
-                }
-            };
+                listener.onFailure(e);
+            }
+        }
+
+        private AbstractRunnable fillGapRunnable(SparseFileTracker.Gap gap, RangeMissingHandler writer, ActionListener<Void> listener) {
+            return ActionRunnable.run(listener.delegateResponse((l, e) -> failGapAndListener(gap, l, e)), () -> {
+                var ioRef = io;
+                assert regionOwners.get(ioRef) == CacheFileRegion.this;
+                assert CacheFileRegion.this.hasReferences() : CacheFileRegion.this;
+                int start = Math.toIntExact(gap.start());
+                writer.fillCacheRange(
+                    ioRef,
+                    start,
+                    start,
+                    Math.toIntExact(gap.end() - start),
+                    progress -> gap.onProgress(start + progress)
+                );
+                writeCount.increment();
+                gap.onCompletion();
+            });
         }
 
-        private static void releaseAndFail(ActionListener<?> listener, Releasable decrementRef, Exception e) {
+        private static void failGapAndListener(SparseFileTracker.Gap gap, ActionListener<?> listener, Exception e) {
             try {
-                Releasables.close(decrementRef);
+                gap.onFailure(e);
             } catch (Exception ex) {
                 e.addSuppressed(ex);
             }

From f11ef44084f71bf144e720a987686ee164bfeff0 Mon Sep 17 00:00:00 2001
From: Salvatore Campagna
 <93581129+salvatore-campagna@users.noreply.github.com>
Date: Thu, 30 May 2024 14:26:48 +0200
Subject: [PATCH 056/208] Introduce `logs` index mode as Tech Preview (#108896)

This PR introduces a new index mode, `logs`, which enables usage of LogsDB in Elasticsearch.
As a result of adopting the `logs` index mode, default index sorting is applied using the hostname
and @timestamp fields. Users are allowed, anyway, to override index sort settings.
By default, it will also use synthetic source and the same codecs used by TSDB.

Note: the logs index mode is a Tech Preview feature.
---
 .../src/main/resources/changelog-schema.json  |   1 +
 docs/changelog/108896.yaml                    |   6 +
 docs/reference/index-modules.asciidoc         |  31 +
 .../rest-api-spec/test/logsdb/10_settings.yml | 598 ++++++++++++++++++
 .../org/elasticsearch/index/IndexMode.java    | 115 +++-
 .../elasticsearch/index/IndexSortConfig.java  |  15 +-
 .../index/codec/PerFieldFormatSupplier.java   |   8 +-
 .../indices/CreateIndexCapabilities.java      |  24 +
 .../admin/indices/RestCreateIndexAction.java  |   6 +
 .../index/LogsIndexModeTests.java             |  74 +++
 .../index/engine/FollowingEngineTests.java    |   3 +
 .../xpack/esql/plan/physical/EsQueryExec.java |   2 +-
 12 files changed, 871 insertions(+), 12 deletions(-)
 create mode 100644 docs/changelog/108896.yaml
 create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
 create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java

diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json
index 8bdfb8d6c0b9c..a38eb32062146 100644
--- a/build-tools-internal/src/main/resources/changelog-schema.json
+++ b/build-tools-internal/src/main/resources/changelog-schema.json
@@ -68,6 +68,7 @@
             "Java High Level REST Client",
             "Java Low Level REST Client",
             "License",
+            "Logs",
             "Machine Learning",
             "Mapping",
             "Monitoring",
diff --git a/docs/changelog/108896.yaml b/docs/changelog/108896.yaml
new file mode 100644
index 0000000000000..c52f074b65605
--- /dev/null
+++ b/docs/changelog/108896.yaml
@@ -0,0 +1,6 @@
+pr: 108896
+summary: Introduce `logs` index mode as Tech Preview
+area: Logs
+type: feature
+issues:
+ - 108896
diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index 1b6914e946c82..e826956440497 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -82,6 +82,37 @@ breaking change].
     after segments are merged. Segment merging can be forced using
     <<indices-forcemerge,force merge>>.
 
+[[index-mode-setting]] `index.mode`::
++
+The `index.mode` setting is used to control settings applied in specific domains like ingestion of time series data or logs.
+Different mutually exclusive modes exist, which are used to apply settings or default values controlling indexing of documents,
+sorting and other parameters whose value affects indexing or query performance.
++
+[source,console]
+----------------
+PUT my-index-000001
+{
+  "settings": {
+    "index":{
+      "mode":"standard" <1>
+    }
+  }
+}
+----------------
++
+<1> This index uses the `standard` index mode
++
+Index mode supports the following values:
+
+`null`::: Default value (same as `standard`).
+
+`standard`::: Standard indexing with default settings.
+
+`time_series`::: Index mode optimized for storage of metrics documented in <<tsds-index-settings,TSDS Settings>>.
+
+`logs`::: Index mode optimized for storage of logs. It applies default sort settings on the `hostname` and `timestamp` fields and uses <<synthetic-source,synthetic `_source`>>. <<index-modules-index-sorting,Index sorting>> on different fields is still allowed.
+preview:[]
+
 [[routing-partition-size]] `index.routing_partition_size`::
 
     The number of shards a custom <<mapping-routing-field,routing>> value can go to.
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
new file mode 100644
index 0000000000000..95075da20fe5e
--- /dev/null
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
@@ -0,0 +1,598 @@
+---
+setup:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [logs_index_mode]
+      reason: "Support for 'logs' index mode capability required"
+
+---
+create logs index:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mode: logs
+              number_of_replicas: 0
+              number_of_shards: 2
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "hostname": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "message": "No, I am your father." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "hostname": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "message": "Do. Or do not. There is no try." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "hostname": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "message": "May the force be with you." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "hostname": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "message": "I find your lack of faith disturbing." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "hostname": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "message": "Wars not make one great." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "hostname": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "message": "That's no moon. It's a space station." }
+
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 6 }
+
+  - do:
+      indices.get_settings:
+        index: test
+
+  - is_true: test
+  - match: { test.settings.index.mode: "logs" }
+
+
+---
+using default timestamp field mapping:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      indices.create:
+        index: test-timestamp-missing
+        body:
+          settings:
+            index:
+              mode: logs
+              number_of_replicas: 0
+              number_of_shards: 2
+          mappings:
+            properties:
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+---
+missing hostname field:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test-hostname-missing
+        body:
+          settings:
+            index:
+              mode: logs
+              number_of_replicas: 0
+              number_of_shards: 2
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "unknown index sort field:[hostname]" }
+
+---
+missing sort field:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test-hostname-missing
+        body:
+          settings:
+            index:
+              mode: logs
+              number_of_replicas: 0
+              number_of_shards: 2
+              sort:
+                field: [ "host_name" ]
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "unknown index sort field:[host_name]" }
+
+---
+non-default sort settings:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      indices.create:
+        index: test-sort
+        body:
+          settings:
+
+            index:
+              mode: logs
+              number_of_shards: 2
+              number_of_replicas: 0
+              sort:
+                field: [ "agent_id", "@timestamp" ]
+                order: [ "asc", "desc" ]
+                missing: [ "_last", "_first" ]
+                mode: [ "max", "max" ]
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - do:
+      indices.get_settings:
+        index: test-sort
+
+  - is_true: test-sort
+  - match: { test-sort.settings.index.mode: "logs" }
+  - match: { test-sort.settings.index.sort.field.0: "agent_id" }
+  - match: { test-sort.settings.index.sort.field.1: "@timestamp" }
+  - match: { test-sort.settings.index.sort.order.0: "asc" }
+  - match: { test-sort.settings.index.sort.order.1: "desc" }
+  - match: { test-sort.settings.index.sort.missing.0: "_last" }
+  - match: { test-sort.settings.index.sort.missing.1: "_first" }
+  - match: { test-sort.settings.index.sort.mode.0: "max" }
+  - match: { test-sort.settings.index.sort.mode.1: "max" }
+
+---
+override sort order settings:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      indices.create:
+        index: test-sort-order
+        body:
+          settings:
+
+            index:
+              mode: logs
+              number_of_shards: 2
+              number_of_replicas: 0
+              sort:
+                order: [ "asc", "asc" ]
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - do:
+      indices.get_settings:
+        index: test-sort-order
+
+  - is_true: test-sort-order
+  - match: { test-sort-order.settings.index.mode: "logs" }
+  - match: { test-sort-order.settings.index.sort.field.0: null }
+  - match: { test-sort-order.settings.index.sort.field.1: null }
+  - match: { test-sort-order.settings.index.sort.order.0: "asc" }
+  - match: { test-sort-order.settings.index.sort.order.1: "asc" }
+
+---
+override sort missing settings:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      indices.create:
+        index: test-sort-missing
+        body:
+          settings:
+
+            index:
+              mode: logs
+              number_of_shards: 2
+              number_of_replicas: 0
+              sort:
+                missing: [ "_last", "_first" ]
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - do:
+      indices.get_settings:
+        index: test-sort-missing
+
+  - is_true: test-sort-missing
+  - match: { test-sort-missing.settings.index.mode: "logs" }
+  - match: { test-sort-missing.settings.index.sort.field.0: null }
+  - match: { test-sort-missing.settings.index.sort.field.1: null }
+  - match: { test-sort-missing.settings.index.sort.missing.0: "_last" }
+  - match: { test-sort-missing.settings.index.sort.missing.1: "_first" }
+
+---
+override sort mode settings:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      indices.create:
+        index: test-sort-mode
+        body:
+          settings:
+
+            index:
+              mode: logs
+              number_of_shards: 2
+              number_of_replicas: 0
+              sort:
+                mode: [ "max", "max" ]
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - do:
+      indices.get_settings:
+        index: test-sort-mode
+
+  - is_true: test-sort-mode
+  - match: { test-sort-mode.settings.index.mode: "logs" }
+  - match: { test-sort-mode.settings.index.sort.field.0: null }
+  - match: { test-sort-mode.settings.index.sort.field.1: null }
+  - match: { test-sort-mode.settings.index.sort.mode.0: "max" }
+  - match: { test-sort-mode.settings.index.sort.mode.1: "max" }
+
+---
+override sort field using nested field type in sorting:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test-nested-sort
+        body:
+          settings:
+            index:
+              mode: logs
+              number_of_replicas: 0
+              number_of_shards: 2
+              sort:
+                field: [ "hostname", "nested", "@timestamp" ]
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+              nested:
+                type: nested
+                properties:
+                  keywords:
+                    type: keyword
+
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "cannot have nested fields when index sort is activated" }
+
+---
+override sort field using nested field type:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test-nested
+        body:
+          settings:
+            index:
+              mode: logs
+              number_of_replicas: 0
+              number_of_shards: 2
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+              nested:
+                type: nested
+                properties:
+                  keywords:
+                    type: keyword
+
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "cannot have nested fields when index sort is activated" }
+
+---
+routing path not allowed in logs mode:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mode: logs
+              number_of_replicas: 0
+              number_of_shards: 2
+              routing_path: [ "hostname", "agent_id" ]
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "[index.routing_path] requires [index.mode=time_series]" }
+
+---
+start time not allowed in logs mode:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mode: logs
+              number_of_replicas: 0
+              number_of_shards: 2
+              time_series:
+                start_time: 2023-01-01T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "[index.time_series.start_time] requires [index.mode=time_series]" }
+
+---
+end time not allowed in logs mode:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logs_index_mode ]
+      reason: "Support for 'logs' index mode capability required"
+
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mode: logs
+              number_of_replicas: 0
+              number_of_shards: 2
+              time_series:
+                end_time: 2023-01-30T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              hostname:
+                type: keyword
+              agent_id:
+                type: keyword
+              process_id:
+                type: integer
+              http_method:
+                type: keyword
+              message:
+                type: text
+
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "[index.time_series.end_time] requires [index.mode=time_series]" }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java
index 05169836d6617..3df5b3fe288a2 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexMode.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java
@@ -53,15 +53,7 @@ public enum IndexMode {
     STANDARD("standard") {
         @Override
         void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
-            settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH);
-            settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME);
-            settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_END_TIME);
-        }
-
-        private static void settingRequiresTimeSeries(Map<Setting<?>, Object> settings, Setting<?> setting) {
-            if (false == Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting))) {
-                throw new IllegalArgumentException("[" + setting.getKey() + "] requires " + tsdbMode());
-            }
+            IndexMode.validateTimeSeriesSettings(settings);
         }
 
         @Override
@@ -225,12 +217,95 @@ public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
             }
         }
 
+        @Override
+        public boolean isSyntheticSourceEnabled() {
+            return true;
+        }
+    },
+    LOGS("logs") {
+        @Override
+        void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
+            IndexMode.validateTimeSeriesSettings(settings);
+        }
+
+        @Override
+        public void validateMapping(MappingLookup lookup) {}
+
+        @Override
+        public void validateAlias(String indexRouting, String searchRouting) {
+
+        }
+
+        @Override
+        public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) throws IOException {
+            if (isDataStream) {
+                MetadataCreateDataStreamService.validateTimestampFieldMapping(mappingLookup);
+            }
+        }
+
+        @Override
+        public CompressedXContent getDefaultMapping() {
+            return DEFAULT_LOGS_TIMESTAMP_MAPPING;
+        }
+
+        @Override
+        public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) {
+            return new ProvidedIdFieldMapper(fieldDataEnabled);
+        }
+
+        @Override
+        public IdFieldMapper idFieldMapperWithoutFieldData() {
+            return ProvidedIdFieldMapper.NO_FIELD_DATA;
+        }
+
+        @Override
+        public TimestampBounds getTimestampBound(IndexMetadata indexMetadata) {
+            return null;
+        }
+
+        @Override
+        public MetadataFieldMapper timeSeriesIdFieldMapper() {
+            // non time-series indices must not have a TimeSeriesIdFieldMapper
+            return null;
+        }
+
+        @Override
+        public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() {
+            // non time-series indices must not have a TimeSeriesRoutingIdFieldMapper
+            return null;
+        }
+
+        @Override
+        public DocumentDimensions buildDocumentDimensions(IndexSettings settings) {
+            return new DocumentDimensions.OnlySingleValueAllowed();
+        }
+
+        @Override
+        public boolean shouldValidateTimestamp() {
+            return false;
+        }
+
+        @Override
+        public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {}
+
         @Override
         public boolean isSyntheticSourceEnabled() {
             return true;
         }
     };
 
+    private static void validateTimeSeriesSettings(Map<Setting<?>, Object> settings) {
+        settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH);
+        settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME);
+        settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_END_TIME);
+    }
+
+    private static void settingRequiresTimeSeries(Map<Setting<?>, Object> settings, Setting<?> setting) {
+        if (false == Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting))) {
+            throw new IllegalArgumentException("[" + setting.getKey() + "] requires " + tsdbMode());
+        }
+    }
+
     protected static String tsdbMode() {
         return "[" + IndexSettings.MODE.getKey() + "=time_series]";
     }
@@ -257,6 +332,27 @@ protected static String tsdbMode() {
         }
     }
 
+    public static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING;
+
+    static {
+        try {
+            DEFAULT_LOGS_TIMESTAMP_MAPPING = new CompressedXContent(
+                ((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
+                    .startObject(DataStreamTimestampFieldMapper.NAME)
+                    .field("enabled", true)
+                    .endObject()
+                    .startObject("properties")
+                    .startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
+                    .field("type", DateFieldMapper.CONTENT_TYPE)
+                    .endObject()
+                    .endObject()
+                    .endObject())
+            );
+        } catch (IOException e) {
+            throw new AssertionError(e);
+        }
+    }
+
     private static final List<Setting<?>> TIME_SERIES_UNSUPPORTED = List.of(
         IndexSortConfig.INDEX_SORT_FIELD_SETTING,
         IndexSortConfig.INDEX_SORT_ORDER_SETTING,
@@ -368,6 +464,7 @@ public static IndexMode fromString(String value) {
         return switch (value) {
             case "standard" -> IndexMode.STANDARD;
             case "time_series" -> IndexMode.TIME_SERIES;
+            case "logs" -> IndexMode.LOGS;
             default -> throw new IllegalArgumentException(
                 "["
                     + value
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java
index 98c2e31838379..74c2c57594e72 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java
@@ -12,6 +12,7 @@
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.search.SortedSetSortField;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Setting;
@@ -152,10 +153,16 @@ public IndexSortConfig(IndexSettings indexSettings) {
         }
 
         List<String> fields = INDEX_SORT_FIELD_SETTING.get(settings);
-        this.sortSpecs = fields.stream().map((name) -> new FieldSortSpec(name)).toArray(FieldSortSpec[]::new);
+        if (this.indexMode == IndexMode.LOGS && fields.isEmpty()) {
+            fields = List.of("hostname", DataStream.TIMESTAMP_FIELD_NAME);
+        }
+        this.sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new);
 
         if (INDEX_SORT_ORDER_SETTING.exists(settings)) {
             List<SortOrder> orders = INDEX_SORT_ORDER_SETTING.get(settings);
+            if (this.indexMode == IndexMode.LOGS && orders.isEmpty()) {
+                orders = List.of(SortOrder.DESC, SortOrder.DESC);
+            }
             if (orders.size() != sortSpecs.length) {
                 throw new IllegalArgumentException(
                     "index.sort.field:" + fields + " index.sort.order:" + orders.toString() + ", size mismatch"
@@ -168,6 +175,9 @@ public IndexSortConfig(IndexSettings indexSettings) {
 
         if (INDEX_SORT_MODE_SETTING.exists(settings)) {
             List<MultiValueMode> modes = INDEX_SORT_MODE_SETTING.get(settings);
+            if (this.indexMode == IndexMode.LOGS && modes.isEmpty()) {
+                modes = List.of(MultiValueMode.MIN, MultiValueMode.MIN);
+            }
             if (modes.size() != sortSpecs.length) {
                 throw new IllegalArgumentException("index.sort.field:" + fields + " index.sort.mode:" + modes + ", size mismatch");
             }
@@ -178,6 +188,9 @@ public IndexSortConfig(IndexSettings indexSettings) {
 
         if (INDEX_SORT_MISSING_SETTING.exists(settings)) {
             List<String> missingValues = INDEX_SORT_MISSING_SETTING.get(settings);
+            if (this.indexMode == IndexMode.LOGS && missingValues.isEmpty()) {
+                missingValues = List.of("_first", "_first");
+            }
             if (missingValues.size() != sortSpecs.length) {
                 throw new IllegalArgumentException(
                     "index.sort.field:" + fields + " index.sort.missing:" + missingValues + ", size mismatch"
diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java
index 81fc2c0b4a065..0b4bb9dfc10ae 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java
@@ -107,7 +107,9 @@ boolean useTSDBDocValuesFormat(final String field) {
             return false;
         }
 
-        return mapperService != null && isTimeSeriesModeIndex() && mapperService.getIndexSettings().isES87TSDBCodecEnabled();
+        return mapperService != null
+            && (isTimeSeriesModeIndex() || isLogsModeIndex())
+            && mapperService.getIndexSettings().isES87TSDBCodecEnabled();
     }
 
     private boolean excludeFields(String fieldName) {
@@ -120,4 +122,8 @@ private boolean isTimeSeriesModeIndex() {
         return mapperService != null && IndexMode.TIME_SERIES == mapperService.getIndexSettings().getMode();
     }
 
+    private boolean isLogsModeIndex() {
+        return mapperService != null && IndexMode.LOGS == mapperService.getIndexSettings().getMode();
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java
new file mode 100644
index 0000000000000..700baac09865e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/CreateIndexCapabilities.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.rest.action.admin.indices;
+
+import java.util.Set;
+
+/**
+ * A {@link Set} of "capabilities" supported by the {@link RestCreateIndexAction}.
+ */
+public class CreateIndexCapabilities {
+
+    /**
+     * Support for using the 'logs' index mode.
+     */
+    private static final String LOGS_INDEX_MODE_CAPABILITY = "logs_index_mode";
+
+    public static Set<String> CAPABILITIES = Set.of(LOGS_INDEX_MODE_CAPABILITY);
+}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
index 3ef96bd44a597..4f9fd9d03521d 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
@@ -28,6 +28,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
@@ -145,4 +146,9 @@ static Map<String, Object> prepareMappings(Map<String, Object> source) {
         newSource.put("mappings", singletonMap(MapperService.SINGLE_MAPPING_NAME, mappings));
         return newSource;
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return CreateIndexCapabilities.CAPABILITIES;
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
new file mode 100644
index 0000000000000..fd73a8c9f8f52
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.MapperServiceTestCase;
+import org.hamcrest.Matchers;
+
+public class LogsIndexModeTests extends MapperServiceTestCase {
+    public void testLogsIndexModeSetting() {
+        assertThat(IndexSettings.MODE.get(buildSettings()), Matchers.equalTo(IndexMode.LOGS));
+    }
+
+    public void testSortField() {
+        final Settings sortSettings = Settings.builder()
+            .put(buildSettings())
+            .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
+            .build();
+        final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
+        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+    }
+
+    public void testSortMode() {
+        final Settings sortSettings = Settings.builder()
+            .put(buildSettings())
+            .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
+            .put(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "max")
+            .build();
+        final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
+        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("max", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey())));
+    }
+
+    public void testSortOrder() {
+        final Settings sortSettings = Settings.builder()
+            .put(buildSettings())
+            .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
+            .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc")
+            .build();
+        final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
+        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("desc", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey())));
+    }
+
+    public void testSortMissing() {
+        final Settings sortSettings = Settings.builder()
+            .put(buildSettings())
+            .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
+            .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last")
+            .build();
+        final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
+        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("_last", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey())));
+    }
+
+    private Settings buildSettings() {
+        return Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGS.getName()).build();
+    }
+
+    private String getIndexSetting(final IndexSettings settings, final String name) {
+        return settings.getIndexMetadata().getSettings().get(name);
+    }
+}
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
index 0abfc6e911d2d..15e358b68e648 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
@@ -696,6 +696,9 @@ public void testProcessOnceOnPrimary() throws Exception {
             case TIME_SERIES:
                 settingsBuilder.put("index.mode", "time_series").put("index.routing_path", "foo");
                 break;
+            case LOGS:
+                settingsBuilder.put("index.mode", "logs");
+                break;
             default:
                 throw new UnsupportedOperationException("Unknown index mode [" + indexMode + "]");
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
index b354e519c0b99..9f9e192036226 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
@@ -86,7 +86,7 @@ public EsQueryExec(
 
     private static List<Attribute> sourceAttributes(Source source, IndexMode indexMode) {
         return switch (indexMode) {
-            case STANDARD -> List.of(new FieldAttribute(source, DOC_ID_FIELD.getName(), DOC_ID_FIELD));
+            case STANDARD, LOGS -> List.of(new FieldAttribute(source, DOC_ID_FIELD.getName(), DOC_ID_FIELD));
             case TIME_SERIES -> List.of(
                 new FieldAttribute(source, DOC_ID_FIELD.getName(), DOC_ID_FIELD),
                 new FieldAttribute(source, TSID_FIELD.getName(), TSID_FIELD),

From f5cc9908f69c1171bdb04086c6d3a378e753df75 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 30 May 2024 08:29:34 -0400
Subject: [PATCH 057/208] ESQL: Move serialization of DataType (#109184)

This moves the serialization of `DataType` from `PlanNamedTypes` and
into `DataType` itself to better align with the rest of Elasticsearch.
---
 .../xpack/esql/core/type/DataType.java        | 23 ++++++++++++++++++-
 .../xpack/esql/core/type/DataTypes.java       |  3 +++
 .../xpack/esql/io/stream/PlanNamedTypes.java  | 10 ++++----
 .../xpack/esql/io/stream/PlanStreamInput.java | 16 -------------
 .../xpack/esql/plan/physical/EsQueryExec.java |  8 ++-----
 .../xpack/esql/planner/PlannerUtils.java      |  5 ++--
 6 files changed, 34 insertions(+), 31 deletions(-)

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
index d1749ac3bf3e8..1dacc394c8d21 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
@@ -7,12 +7,16 @@
 
 package org.elasticsearch.xpack.esql.core.type;
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.search.sort.ScriptSortBuilder;
 
+import java.io.IOException;
 import java.util.Locale;
 import java.util.Objects;
 
-public class DataType {
+public class DataType implements Writeable {
 
     private final String typeName;
 
@@ -118,4 +122,21 @@ public boolean equals(Object obj) {
     public String toString() {
         return name;
     }
+
+    public static DataType readFrom(StreamInput in) throws IOException {
+        String name = in.readString();
+        if (name.equalsIgnoreCase(DataTypes.DOC_DATA_TYPE.name())) {
+            return DataTypes.DOC_DATA_TYPE;
+        }
+        DataType dataType = DataTypes.fromTypeName(name);
+        if (dataType == null) {
+            throw new IOException("Unknown DataType for type name: " + name);
+        }
+        return dataType;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(typeName);
+    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java
index c28bf96728ad0..8d4129d1abbcc 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java
@@ -78,6 +78,9 @@ public final class DataTypes {
     public static final DataType COUNTER_INTEGER = new DataType("counter_integer", Integer.BYTES, false, false, true);
     public static final DataType COUNTER_DOUBLE = new DataType("counter_double", Double.BYTES, false, false, true);
 
+    public static final DataType DOC_DATA_TYPE = new DataType("_doc", Integer.BYTES * 3, false, false, false);
+    public static final DataType TSID_DATA_TYPE = new DataType("_tsid", Integer.MAX_VALUE, false, false, true);
+
     private static final Collection<DataType> TYPES = Stream.of(
         UNSUPPORTED,
         NULL,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index 33c5937c0fd0a..7a7cbb55f5629 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -1070,7 +1070,7 @@ static FieldAttribute readFieldAttribute(PlanStreamInput in) throws IOException
             Source.readFrom(in),
             in.readOptionalWithReader(PlanNamedTypes::readFieldAttribute),
             in.readString(),
-            in.dataTypeFromTypeName(in.readString()),
+            DataType.readFrom(in),
             in.readEsFieldNamed(),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
@@ -1095,7 +1095,7 @@ static ReferenceAttribute readReferenceAttr(PlanStreamInput in) throws IOExcepti
         return new ReferenceAttribute(
             Source.readFrom(in),
             in.readString(),
-            in.dataTypeFromTypeName(in.readString()),
+            DataType.readFrom(in),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
             NameId.readFrom(in),
@@ -1117,7 +1117,7 @@ static MetadataAttribute readMetadataAttr(PlanStreamInput in) throws IOException
         return new MetadataAttribute(
             Source.readFrom(in),
             in.readString(),
-            in.dataTypeFromTypeName(in.readString()),
+            DataType.readFrom(in),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
             NameId.readFrom(in),
@@ -1160,7 +1160,7 @@ static void writeUnsupportedAttr(PlanStreamOutput out, UnsupportedAttribute unsu
     static EsField readEsField(PlanStreamInput in) throws IOException {
         return new EsField(
             in.readString(),
-            in.dataTypeFromTypeName(in.readString()),
+            DataType.readFrom(in),
             in.readImmutableMap(StreamInput::readString, readerFromPlanReader(PlanStreamInput::readEsFieldNamed)),
             in.readBoolean(),
             in.readBoolean()
@@ -1875,7 +1875,7 @@ static void writeAlias(PlanStreamOutput out, Alias alias) throws IOException {
     static Literal readLiteral(PlanStreamInput in) throws IOException {
         Source source = Source.readFrom(in);
         Object value = in.readGenericValue();
-        DataType dataType = in.dataTypeFromTypeName(in.readString());
+        DataType dataType = DataType.readFrom(in);
         return new Literal(source, mapToLiteralValue(in, dataType, value), dataType);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
index d5b1990e69c5f..6c1b7db24fe46 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
@@ -30,14 +30,11 @@
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanNamedReader;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader;
-import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -90,19 +87,6 @@ public PlanStreamInput(
         this.nameIdFunction = new NameIdMapper();
     }
 
-    DataType dataTypeFromTypeName(String typeName) throws IOException {
-        DataType dataType;
-        if (typeName.equalsIgnoreCase(EsQueryExec.DOC_DATA_TYPE.name())) {
-            dataType = EsQueryExec.DOC_DATA_TYPE;
-        } else {
-            dataType = EsqlDataTypes.fromTypeName(typeName);
-        }
-        if (dataType == null) {
-            throw new IOException("Unknown DataType for type name: " + typeName);
-        }
-        return dataType;
-    }
-
     public LogicalPlan readLogicalPlanNode() throws IOException {
         return readNamed(LogicalPlan.class);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
index 9f9e192036226..30986f9c626da 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.NodeUtils;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 
@@ -29,11 +28,8 @@
 import java.util.Objects;
 
 public class EsQueryExec extends LeafExec implements EstimatesRowSize {
-    public static final DataType DOC_DATA_TYPE = new DataType("_doc", Integer.BYTES * 3, false, false, false);
-    public static final DataType TSID_DATA_TYPE = new DataType("_tsid", Integer.MAX_VALUE, false, false, true);
-
-    static final EsField DOC_ID_FIELD = new EsField("_doc", DOC_DATA_TYPE, Map.of(), false);
-    static final EsField TSID_FIELD = new EsField("_tsid", TSID_DATA_TYPE, Map.of(), true);
+    static final EsField DOC_ID_FIELD = new EsField("_doc", DataTypes.DOC_DATA_TYPE, Map.of(), false);
+    static final EsField TSID_FIELD = new EsField("_tsid", DataTypes.TSID_DATA_TYPE, Map.of(), true);
     static final EsField TIMESTAMP_FIELD = new EsField("@timestamp", DataTypes.DATETIME, Map.of(), true);
     static final EsField INTERVAL_FIELD = new EsField("@timestamp_interval", DataTypes.DATETIME, Map.of(), true);
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
index 0b4ee46af69ee..39b641a872c58 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.TopN;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
-import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.esql.plan.physical.EsSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize;
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec;
@@ -268,10 +267,10 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field
         if (dataType == DataTypes.BOOLEAN) {
             return ElementType.BOOLEAN;
         }
-        if (dataType == EsQueryExec.DOC_DATA_TYPE) {
+        if (dataType == DataTypes.DOC_DATA_TYPE) {
             return ElementType.DOC;
         }
-        if (dataType == EsQueryExec.TSID_DATA_TYPE) {
+        if (dataType == DataTypes.TSID_DATA_TYPE) {
             return ElementType.BYTES_REF;
         }
         if (EsqlDataTypes.isSpatialPoint(dataType)) {

From dfbc1a165b4aa789451eb561fa9fba6a04e65ad9 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Thu, 30 May 2024 13:44:37 +0100
Subject: [PATCH 058/208] Improve threading in
 SecurityNetty4HttpServerTransportTests (#109199)

These tests interact with (embedded) Netty channels from multiple
threads, which is rather different from the production threading model
and blocks some upcoming work that tightens up the requirements in this
area. However, we can't use the test thread for all the interactions
because some of them assert they are running on a network thread, so
with this change we move more of the work onto (fake) network threads.
---
 ...ecurityNetty4HttpServerTransportTests.java | 558 +++++++++---------
 1 file changed, 266 insertions(+), 292 deletions(-)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
index a9d8c1dfc8d9e..0663172fa2e9c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
@@ -323,19 +323,20 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 }
             )
         ) {
-            final ChannelHandler handler = transport.configureServerChannelHandler();
-            final EmbeddedChannel ch = new EmbeddedChannel(handler);
-            // remove these pipeline handlers as they interfere in the test scenario
-            for (String pipelineHandlerName : ch.pipeline().names()) {
-                if (pipelineHandlerName.equals("decoder")
-                    || pipelineHandlerName.equals("encoder")
-                    || pipelineHandlerName.equals("encoder_compress")
-                    || pipelineHandlerName.equals("chunked_writer")) {
-                    ch.pipeline().remove(pipelineHandlerName);
+            safeGet(testThreadPool.generic().submit(() -> {
+                final ChannelHandler handler = transport.configureServerChannelHandler();
+                final EmbeddedChannel ch = new EmbeddedChannel(handler);
+                // remove these pipeline handlers as they interfere in the test scenario
+                for (String pipelineHandlerName : ch.pipeline().names()) {
+                    if (pipelineHandlerName.equals("decoder")
+                        || pipelineHandlerName.equals("encoder")
+                        || pipelineHandlerName.equals("encoder_compress")
+                        || pipelineHandlerName.equals("chunked_writer")) {
+                        ch.pipeline().remove(pipelineHandlerName);
+                    }
                 }
-            }
-            // STEP 0: send a "wrapped" request
-            var writeFuture = testThreadPool.generic().submit(() -> {
+
+                // STEP 0: send a "wrapped" request
                 ch.writeInbound(
                     HttpHeadersAuthenticatorUtils.wrapAsMessageWithAuthenticationContext(
                         new DefaultHttpRequest(HTTP_1_1, HttpMethod.GET, "/wrapped_request")
@@ -343,8 +344,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 );
                 ch.writeInbound(new DefaultLastHttpContent());
                 ch.flushInbound();
-            });
-            writeFuture.get();
+            }));
+
             // STEP 3: assert the wrapped context
             var storedAuthnContext = HttpHeadersAuthenticatorUtils.extractAuthenticationContext(dispatchedHttpRequestReference.get());
             assertThat(storedAuthnContext, notNullValue());
@@ -378,35 +379,33 @@ public void testHttpHeaderAuthnBypassHeaderValidator() throws Exception {
                 (httpPreRequest, channel, listener) -> listener.onResponse(null)
             )
         ) {
-            final ChannelHandler handler = transport.configureServerChannelHandler();
-            final EmbeddedChannel ch = new EmbeddedChannel(handler);
-            for (String pipelineHandlerName : ch.pipeline().names()) {
-                // remove the decoder AND the header_validator
-                if (pipelineHandlerName.equals("decoder") || pipelineHandlerName.equals("header_validator")
-                // remove these pipeline handlers as they interfere in the test scenario
-                    || pipelineHandlerName.equals("encoder")
-                    || pipelineHandlerName.equals("encoder_compress")
-                    || pipelineHandlerName.equals("chunked_writer")) {
-                    ch.pipeline().remove(pipelineHandlerName);
+            safeGet(testThreadPool.generic().submit(() -> {
+
+                final ChannelHandler handler = transport.configureServerChannelHandler();
+                final EmbeddedChannel ch = new EmbeddedChannel(handler);
+                for (String pipelineHandlerName : ch.pipeline().names()) {
+                    // remove the decoder AND the header_validator
+                    if (pipelineHandlerName.equals("decoder") || pipelineHandlerName.equals("header_validator")
+                    // remove these pipeline handlers as they interfere in the test scenario
+                        || pipelineHandlerName.equals("encoder")
+                        || pipelineHandlerName.equals("encoder_compress")
+                        || pipelineHandlerName.equals("chunked_writer")) {
+                        ch.pipeline().remove(pipelineHandlerName);
+                    }
                 }
-            }
-            // this tests a request that cannot be authenticated, but somehow passed authentication
-            // this is the case of an erroneous internal state
-            var writeFuture = testThreadPool.generic().submit(() -> {
+                // this tests a request that cannot be authenticated, but somehow passed authentication
+                // this is the case of an erroneous internal state
                 ch.writeInbound(new DefaultFullHttpRequest(HTTP_1_1, HttpMethod.GET, "/unauthenticable_request"));
                 ch.flushInbound();
-            });
-            writeFuture.get();
-            ch.flushOutbound();
-            Netty4FullHttpResponse response = ch.readOutbound();
-            assertThat(response.status(), is(HttpResponseStatus.INTERNAL_SERVER_ERROR));
-            String responseContentString = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8);
-            assertThat(
-                responseContentString,
-                containsString("\"type\":\"security_exception\",\"reason\":\"Request is not authenticated\"")
-            );
-            // this tests a request that CAN be authenticated, but that, somehow, has not been
-            writeFuture = testThreadPool.generic().submit(() -> {
+                ch.flushOutbound();
+                Netty4FullHttpResponse response = ch.readOutbound();
+                assertThat(response.status(), is(HttpResponseStatus.INTERNAL_SERVER_ERROR));
+                String responseContentString = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8);
+                assertThat(
+                    responseContentString,
+                    containsString("\"type\":\"security_exception\",\"reason\":\"Request is not authenticated\"")
+                );
+                // this tests a request that CAN be authenticated, but that, somehow, has not been
                 ch.writeInbound(
                     HttpHeadersAuthenticatorUtils.wrapAsMessageWithAuthenticationContext(
                         new DefaultHttpRequest(HTTP_1_1, HttpMethod.GET, "/_request")
@@ -414,19 +413,16 @@ public void testHttpHeaderAuthnBypassHeaderValidator() throws Exception {
                 );
                 ch.writeInbound(new DefaultLastHttpContent());
                 ch.flushInbound();
-            });
-            writeFuture.get();
-            ch.flushOutbound();
-            response = ch.readOutbound();
-            assertThat(response.status(), is(HttpResponseStatus.INTERNAL_SERVER_ERROR));
-            responseContentString = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8);
-            assertThat(
-                responseContentString,
-                containsString("\"type\":\"security_exception\",\"reason\":\"Request is not authenticated\"")
-            );
-            // this tests the case where authentication passed and the request is to be dispatched, BUT that the authentication context
-            // cannot be instated before dispatching the request
-            writeFuture = testThreadPool.generic().submit(() -> {
+                ch.flushOutbound();
+                response = ch.readOutbound();
+                assertThat(response.status(), is(HttpResponseStatus.INTERNAL_SERVER_ERROR));
+                responseContentString = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8);
+                assertThat(
+                    responseContentString,
+                    containsString("\"type\":\"security_exception\",\"reason\":\"Request is not authenticated\"")
+                );
+                // this tests the case where authentication passed and the request is to be dispatched, BUT that the authentication context
+                // cannot be instated before dispatching the request
                 HttpMessage authenticableMessage = HttpHeadersAuthenticatorUtils.wrapAsMessageWithAuthenticationContext(
                     new DefaultHttpRequest(HTTP_1_1, HttpMethod.GET, "/unauthenticated_request")
                 );
@@ -436,13 +432,12 @@ public void testHttpHeaderAuthnBypassHeaderValidator() throws Exception {
                 ch.writeInbound(authenticableMessage);
                 ch.writeInbound(new DefaultLastHttpContent());
                 ch.flushInbound();
-            });
-            writeFuture.get();
-            ch.flushOutbound();
-            response = ch.readOutbound();
-            assertThat(response.status(), is(HttpResponseStatus.INTERNAL_SERVER_ERROR));
-            responseContentString = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8);
-            assertThat(responseContentString, containsString("\"type\":\"exception\",\"reason\":\"Boom\""));
+                ch.flushOutbound();
+                response = ch.readOutbound();
+                assertThat(response.status(), is(HttpResponseStatus.INTERNAL_SERVER_ERROR));
+                responseContentString = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8);
+                assertThat(responseContentString, containsString("\"type\":\"exception\",\"reason\":\"Boom\""));
+            }));
         } finally {
             testThreadPool.shutdownNow();
         }
@@ -483,43 +478,41 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 (httpPreRequest, channel, listener) -> listener.onResponse(null)
             )
         ) {
-            final ChannelHandler handler = transport.configureServerChannelHandler();
-            final EmbeddedChannel ch = new EmbeddedChannel(handler);
-            // replace the decoder with the vanilla one that does no wrapping and will trip the header validator
-            ch.pipeline().replace("decoder", "decoder", new HttpRequestDecoder());
-            // remove these pipeline handlers as they interfere in the test scenario
-            for (String pipelineHandlerName : ch.pipeline().names()) {
-                if (pipelineHandlerName.equals("encoder")
-                    || pipelineHandlerName.equals("encoder_compress")
-                    || pipelineHandlerName.equals("chunked_writer")) {
-                    ch.pipeline().remove(pipelineHandlerName);
+            safeGet(testThreadPool.generic().submit(() -> {
+                final ChannelHandler handler = transport.configureServerChannelHandler();
+                final EmbeddedChannel ch = new EmbeddedChannel(handler);
+                // replace the decoder with the vanilla one that does no wrapping and will trip the header validator
+                ch.pipeline().replace("decoder", "decoder", new HttpRequestDecoder());
+                // remove these pipeline handlers as they interfere in the test scenario
+                for (String pipelineHandlerName : ch.pipeline().names()) {
+                    if (pipelineHandlerName.equals("encoder")
+                        || pipelineHandlerName.equals("encoder_compress")
+                        || pipelineHandlerName.equals("chunked_writer")) {
+                        ch.pipeline().remove(pipelineHandlerName);
+                    }
                 }
-            }
-            // tests requests that are not wrapped by the "decoder" and so cannot be authenticated
-            testThreadPool.generic().submit(() -> {
+                // tests requests that are not wrapped by the "decoder" and so cannot be authenticated
                 ch.writeInbound(new DefaultFullHttpRequest(HTTP_1_1, HttpMethod.GET, "/unwrapped_full_request"));
                 ch.flushInbound();
-            }).get();
-            ch.flushOutbound();
-            Netty4FullHttpResponse response = ch.readOutbound();
-            assertThat(response.status(), is(HttpResponseStatus.INTERNAL_SERVER_ERROR));
-            var responseContentString = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8);
-            assertThat(
-                responseContentString,
-                containsString("\"type\":\"illegal_state_exception\",\"reason\":\"Cannot authenticate unwrapped requests\"")
-            );
-            testThreadPool.generic().submit(() -> {
+                ch.flushOutbound();
+                Netty4FullHttpResponse response = ch.readOutbound();
+                assertThat(response.status(), is(HttpResponseStatus.INTERNAL_SERVER_ERROR));
+                var responseContentString = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8);
+                assertThat(
+                    responseContentString,
+                    containsString("\"type\":\"illegal_state_exception\",\"reason\":\"Cannot authenticate unwrapped requests\"")
+                );
                 ch.writeInbound(new DefaultHttpRequest(HTTP_1_1, HttpMethod.GET, "/unwrapped_request"));
                 ch.flushInbound();
-            }).get();
-            ch.flushOutbound();
-            response = ch.readOutbound();
-            assertThat(response.status(), is(HttpResponseStatus.INTERNAL_SERVER_ERROR));
-            responseContentString = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8);
-            assertThat(
-                responseContentString,
-                containsString("\"type\":\"illegal_state_exception\",\"reason\":\"Cannot authenticate unwrapped requests\"")
-            );
+                ch.flushOutbound();
+                response = ch.readOutbound();
+                assertThat(response.status(), is(HttpResponseStatus.INTERNAL_SERVER_ERROR));
+                responseContentString = new String(ByteBufUtil.getBytes(response.content()), StandardCharsets.UTF_8);
+                assertThat(
+                    responseContentString,
+                    containsString("\"type\":\"illegal_state_exception\",\"reason\":\"Cannot authenticate unwrapped requests\"")
+                );
+            }));
         } finally {
             testThreadPool.shutdownNow();
         }
@@ -571,111 +564,97 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 }
             )
         ) {
-            final ChannelHandler handler = transport.configureServerChannelHandler();
-            assertThat(authnInvocationCount.get(), is(0));
-            assertThat(badDispatchInvocationCount.get(), is(0));
-            // case 1: invalid initial line
-            {
-                EmbeddedChannel ch = new EmbeddedChannel(handler);
-                ByteBuf buf = ch.alloc().buffer();
-                ByteBufUtil.copy(AsciiString.of("This is not a valid HTTP line"), buf);
-                buf.writeByte(HttpConstants.LF);
-                buf.writeByte(HttpConstants.LF);
-                var writeFuture = testThreadPool.generic().submit(() -> {
+            safeGet(testThreadPool.generic().submit(() -> {
+
+                final ChannelHandler handler = transport.configureServerChannelHandler();
+                assertThat(authnInvocationCount.get(), is(0));
+                assertThat(badDispatchInvocationCount.get(), is(0));
+                // case 1: invalid initial line
+                {
+                    EmbeddedChannel ch = new EmbeddedChannel(handler);
+                    ByteBuf buf = ch.alloc().buffer();
+                    ByteBufUtil.copy(AsciiString.of("This is not a valid HTTP line"), buf);
+                    buf.writeByte(HttpConstants.LF);
+                    buf.writeByte(HttpConstants.LF);
                     ch.writeInbound(buf);
                     ch.flushInbound();
-                });
-                writeFuture.get();
-                assertThat(dispatchThrowableReference.get().toString(), containsString("NOT A VALID HTTP LINE"));
-                assertThat(badDispatchInvocationCount.get(), is(1));
-                assertThat(authnInvocationCount.get(), is(0));
-            }
-            // case 2: too long initial line
-            {
-                EmbeddedChannel ch = new EmbeddedChannel(handler);
-                ByteBuf buf = ch.alloc().buffer();
-                ByteBufUtil.copy(AsciiString.of("GET /this/is/a/valid/but/too/long/initial/line HTTP/1.1"), buf);
-                buf.writeByte(HttpConstants.LF);
-                buf.writeByte(HttpConstants.LF);
-                var writeFuture = testThreadPool.generic().submit(() -> {
+                    assertThat(dispatchThrowableReference.get().toString(), containsString("NOT A VALID HTTP LINE"));
+                    assertThat(badDispatchInvocationCount.get(), is(1));
+                    assertThat(authnInvocationCount.get(), is(0));
+                }
+                // case 2: too long initial line
+                {
+                    EmbeddedChannel ch = new EmbeddedChannel(handler);
+                    ByteBuf buf = ch.alloc().buffer();
+                    ByteBufUtil.copy(AsciiString.of("GET /this/is/a/valid/but/too/long/initial/line HTTP/1.1"), buf);
+                    buf.writeByte(HttpConstants.LF);
+                    buf.writeByte(HttpConstants.LF);
                     ch.writeInbound(buf);
                     ch.flushInbound();
-                });
-                writeFuture.get();
-                assertThat(dispatchThrowableReference.get().toString(), containsString("HTTP line is larger than"));
-                assertThat(badDispatchInvocationCount.get(), is(2));
-                assertThat(authnInvocationCount.get(), is(0));
-            }
-            // case 3: invalid header with no colon
-            {
-                EmbeddedChannel ch = new EmbeddedChannel(handler);
-                ByteBuf buf = ch.alloc().buffer();
-                ByteBufUtil.copy(AsciiString.of("GET /url HTTP/1.1"), buf);
-                buf.writeByte(HttpConstants.LF);
-                ByteBufUtil.copy(AsciiString.of("Host"), buf);
-                buf.writeByte(HttpConstants.LF);
-                buf.writeByte(HttpConstants.LF);
-                var writeFuture = testThreadPool.generic().submit(() -> {
+                    assertThat(dispatchThrowableReference.get().toString(), containsString("HTTP line is larger than"));
+                    assertThat(badDispatchInvocationCount.get(), is(2));
+                    assertThat(authnInvocationCount.get(), is(0));
+                }
+                // case 3: invalid header with no colon
+                {
+                    EmbeddedChannel ch = new EmbeddedChannel(handler);
+                    ByteBuf buf = ch.alloc().buffer();
+                    ByteBufUtil.copy(AsciiString.of("GET /url HTTP/1.1"), buf);
+                    buf.writeByte(HttpConstants.LF);
+                    ByteBufUtil.copy(AsciiString.of("Host"), buf);
+                    buf.writeByte(HttpConstants.LF);
+                    buf.writeByte(HttpConstants.LF);
                     ch.writeInbound(buf);
                     ch.flushInbound();
-                });
-                writeFuture.get();
-                assertThat(dispatchThrowableReference.get().toString(), containsString("No colon found"));
-                assertThat(badDispatchInvocationCount.get(), is(3));
-                assertThat(authnInvocationCount.get(), is(0));
-            }
-            // case 4: invalid header longer than max allowed
-            {
-                EmbeddedChannel ch = new EmbeddedChannel(handler);
-                ByteBuf buf = ch.alloc().buffer();
-                ByteBufUtil.copy(AsciiString.of("GET /url HTTP/1.1"), buf);
-                buf.writeByte(HttpConstants.LF);
-                ByteBufUtil.copy(AsciiString.of("Host: this.looks.like.a.good.url.but.is.longer.than.permitted"), buf);
-                buf.writeByte(HttpConstants.LF);
-                buf.writeByte(HttpConstants.LF);
-                var writeFuture = testThreadPool.generic().submit(() -> {
+                    assertThat(dispatchThrowableReference.get().toString(), containsString("No colon found"));
+                    assertThat(badDispatchInvocationCount.get(), is(3));
+                    assertThat(authnInvocationCount.get(), is(0));
+                }
+                // case 4: invalid header longer than max allowed
+                {
+                    EmbeddedChannel ch = new EmbeddedChannel(handler);
+                    ByteBuf buf = ch.alloc().buffer();
+                    ByteBufUtil.copy(AsciiString.of("GET /url HTTP/1.1"), buf);
+                    buf.writeByte(HttpConstants.LF);
+                    ByteBufUtil.copy(AsciiString.of("Host: this.looks.like.a.good.url.but.is.longer.than.permitted"), buf);
+                    buf.writeByte(HttpConstants.LF);
+                    buf.writeByte(HttpConstants.LF);
                     ch.writeInbound(buf);
                     ch.flushInbound();
-                });
-                writeFuture.get();
-                assertThat(dispatchThrowableReference.get().toString(), containsString("HTTP header is larger than"));
-                assertThat(badDispatchInvocationCount.get(), is(4));
-                assertThat(authnInvocationCount.get(), is(0));
-            }
-            // case 5: invalid header format
-            {
-                EmbeddedChannel ch = new EmbeddedChannel(handler);
-                ByteBuf buf = ch.alloc().buffer();
-                ByteBufUtil.copy(AsciiString.of("GET /url HTTP/1.1"), buf);
-                buf.writeByte(HttpConstants.LF);
-                ByteBufUtil.copy(AsciiString.of("Host: invalid header value"), buf);
-                buf.writeByte(0x01);
-                buf.writeByte(HttpConstants.LF);
-                buf.writeByte(HttpConstants.LF);
-                var writeFuture = testThreadPool.generic().submit(() -> {
+                    assertThat(dispatchThrowableReference.get().toString(), containsString("HTTP header is larger than"));
+                    assertThat(badDispatchInvocationCount.get(), is(4));
+                    assertThat(authnInvocationCount.get(), is(0));
+                }
+                // case 5: invalid header format
+                {
+                    EmbeddedChannel ch = new EmbeddedChannel(handler);
+                    ByteBuf buf = ch.alloc().buffer();
+                    ByteBufUtil.copy(AsciiString.of("GET /url HTTP/1.1"), buf);
+                    buf.writeByte(HttpConstants.LF);
+                    ByteBufUtil.copy(AsciiString.of("Host: invalid header value"), buf);
+                    buf.writeByte(0x01);
+                    buf.writeByte(HttpConstants.LF);
+                    buf.writeByte(HttpConstants.LF);
                     ch.writeInbound(buf);
                     ch.flushInbound();
-                });
-                writeFuture.get();
-                assertThat(dispatchThrowableReference.get().toString(), containsString("Validation failed for header 'Host'"));
-                assertThat(badDispatchInvocationCount.get(), is(5));
-                assertThat(authnInvocationCount.get(), is(0));
-            }
-            // case 6: connection closed before all headers are sent
-            {
-                EmbeddedChannel ch = new EmbeddedChannel(handler);
-                ByteBuf buf = ch.alloc().buffer();
-                ByteBufUtil.copy(AsciiString.of("GET /url HTTP/1.1"), buf);
-                buf.writeByte(HttpConstants.LF);
-                ByteBufUtil.copy(AsciiString.of("Host: localhost"), buf);
-                buf.writeByte(HttpConstants.LF);
-                testThreadPool.generic().submit(() -> {
+                    assertThat(dispatchThrowableReference.get().toString(), containsString("Validation failed for header 'Host'"));
+                    assertThat(badDispatchInvocationCount.get(), is(5));
+                    assertThat(authnInvocationCount.get(), is(0));
+                }
+                // case 6: connection closed before all headers are sent
+                {
+                    EmbeddedChannel ch = new EmbeddedChannel(handler);
+                    ByteBuf buf = ch.alloc().buffer();
+                    ByteBufUtil.copy(AsciiString.of("GET /url HTTP/1.1"), buf);
+                    buf.writeByte(HttpConstants.LF);
+                    ByteBufUtil.copy(AsciiString.of("Host: localhost"), buf);
+                    buf.writeByte(HttpConstants.LF);
                     ch.writeInbound(buf);
                     ch.flushInbound();
-                }).get();
-                testThreadPool.generic().submit(() -> ch.close().get()).get();
-                assertThat(authnInvocationCount.get(), is(0));
-            }
+                    safeGet(ch.close());
+                    assertThat(authnInvocationCount.get(), is(0));
+                }
+            }));
         } finally {
             testThreadPool.shutdownNow();
         }
@@ -717,130 +696,125 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 }
             )
         ) {
-            final ChannelHandler handler = transport.configureServerChannelHandler();
-            final EmbeddedChannel ch = new EmbeddedChannel(handler);
-            // OPTIONS request with fixed length content written in one chunk
-            {
-                ByteBuf buf = ch.alloc().buffer();
-                ByteBufUtil.copy(AsciiString.of("OPTIONS /url/whatever/fixed-length-single-chunk HTTP/1.1"), buf);
-                buf.writeByte(HttpConstants.LF);
-                if (randomBoolean()) {
-                    ByteBufUtil.copy(AsciiString.of("Host: localhost"), buf);
+            safeGet(testThreadPool.generic().submit(() -> {
+                final ChannelHandler handler = transport.configureServerChannelHandler();
+                final EmbeddedChannel ch = new EmbeddedChannel(handler);
+                // OPTIONS request with fixed length content written in one chunk
+                {
+                    ByteBuf buf = ch.alloc().buffer();
+                    ByteBufUtil.copy(AsciiString.of("OPTIONS /url/whatever/fixed-length-single-chunk HTTP/1.1"), buf);
                     buf.writeByte(HttpConstants.LF);
-                }
-                if (randomBoolean()) {
-                    ByteBufUtil.copy(AsciiString.of("Accept: */*"), buf);
+                    if (randomBoolean()) {
+                        ByteBufUtil.copy(AsciiString.of("Host: localhost"), buf);
+                        buf.writeByte(HttpConstants.LF);
+                    }
+                    if (randomBoolean()) {
+                        ByteBufUtil.copy(AsciiString.of("Accept: */*"), buf);
+                        buf.writeByte(HttpConstants.LF);
+                    }
+                    if (randomBoolean()) {
+                        ByteBufUtil.copy(AsciiString.of("Content-Encoding: gzip"), buf);
+                        buf.writeByte(HttpConstants.LF);
+                    }
+                    if (randomBoolean()) {
+                        ByteBufUtil.copy(
+                            AsciiString.of("Content-Type: " + randomFrom("text/plain; charset=utf-8", "application/json; charset=utf-8")),
+                            buf
+                        );
+                        buf.writeByte(HttpConstants.LF);
+                    }
+                    String content = randomAlphaOfLengthBetween(4, 1024);
+                    // having a "Content-Length" request header is what makes it "fixed length"
+                    ByteBufUtil.copy(AsciiString.of("Content-Length: " + content.length()), buf);
                     buf.writeByte(HttpConstants.LF);
-                }
-                if (randomBoolean()) {
-                    ByteBufUtil.copy(AsciiString.of("Content-Encoding: gzip"), buf);
+                    // end of headers
                     buf.writeByte(HttpConstants.LF);
-                }
-                if (randomBoolean()) {
-                    ByteBufUtil.copy(
-                        AsciiString.of("Content-Type: " + randomFrom("text/plain; charset=utf-8", "application/json; charset=utf-8")),
-                        buf
-                    );
-                    buf.writeByte(HttpConstants.LF);
-                }
-                String content = randomAlphaOfLengthBetween(4, 1024);
-                // having a "Content-Length" request header is what makes it "fixed length"
-                ByteBufUtil.copy(AsciiString.of("Content-Length: " + content.length()), buf);
-                buf.writeByte(HttpConstants.LF);
-                // end of headers
-                buf.writeByte(HttpConstants.LF);
-                ByteBufUtil.copy(AsciiString.of(content), buf);
-                // write everything in one single chunk
-                testThreadPool.generic().submit(() -> {
+                    ByteBufUtil.copy(AsciiString.of(content), buf);
+                    // write everything in one single chunk
                     ch.writeInbound(buf);
                     ch.flushInbound();
-                }).get();
-                ch.runPendingTasks();
-                Throwable badRequestCause = badRequestCauseReference.get();
-                assertThat(badRequestCause, instanceOf(HttpHeadersValidationException.class));
-                assertThat(badRequestCause.getCause(), instanceOf(ElasticsearchException.class));
-                assertThat(((ElasticsearchException) badRequestCause.getCause()).status(), is(RestStatus.BAD_REQUEST));
-                assertThat(
-                    ((ElasticsearchException) badRequestCause.getCause()).getDetailedMessage(),
-                    containsString("OPTIONS requests with a payload body are not supported")
-                );
-            }
-            {
-                ByteBuf buf = ch.alloc().buffer();
-                ByteBufUtil.copy(AsciiString.of("OPTIONS /url/whatever/chunked-transfer?encoding HTTP/1.1"), buf);
-                buf.writeByte(HttpConstants.LF);
-                if (randomBoolean()) {
-                    ByteBufUtil.copy(AsciiString.of("Host: localhost"), buf);
-                    buf.writeByte(HttpConstants.LF);
-                }
-                if (randomBoolean()) {
-                    ByteBufUtil.copy(AsciiString.of("Accept: */*"), buf);
-                    buf.writeByte(HttpConstants.LF);
-                }
-                if (randomBoolean()) {
-                    ByteBufUtil.copy(AsciiString.of("Content-Encoding: gzip"), buf);
-                    buf.writeByte(HttpConstants.LF);
-                }
-                if (randomBoolean()) {
-                    ByteBufUtil.copy(
-                        AsciiString.of("Content-Type: " + randomFrom("text/plain; charset=utf-8", "application/json; charset=utf-8")),
-                        buf
+                    ch.runPendingTasks();
+                    Throwable badRequestCause = badRequestCauseReference.get();
+                    assertThat(badRequestCause, instanceOf(HttpHeadersValidationException.class));
+                    assertThat(badRequestCause.getCause(), instanceOf(ElasticsearchException.class));
+                    assertThat(((ElasticsearchException) badRequestCause.getCause()).status(), is(RestStatus.BAD_REQUEST));
+                    assertThat(
+                        ((ElasticsearchException) badRequestCause.getCause()).getDetailedMessage(),
+                        containsString("OPTIONS requests with a payload body are not supported")
                     );
-                    buf.writeByte(HttpConstants.LF);
                 }
-                // do not write a "Content-Length" header to make the request "variable length"
-                if (randomBoolean()) {
-                    ByteBufUtil.copy(AsciiString.of("Transfer-Encoding: " + randomFrom("chunked", "gzip, chunked")), buf);
-                } else {
-                    ByteBufUtil.copy(AsciiString.of("Transfer-Encoding: chunked"), buf);
-                }
-                buf.writeByte(HttpConstants.LF);
-                buf.writeByte(HttpConstants.LF);
-                // maybe append some chunks as well
-                String[] contentParts = randomArray(0, 4, String[]::new, () -> randomAlphaOfLengthBetween(1, 64));
-                for (String content : contentParts) {
-                    ByteBufUtil.copy(AsciiString.of(Integer.toHexString(content.length())), buf);
-                    buf.writeByte(HttpConstants.CR);
+                {
+                    ByteBuf buf = ch.alloc().buffer();
+                    ByteBufUtil.copy(AsciiString.of("OPTIONS /url/whatever/chunked-transfer?encoding HTTP/1.1"), buf);
                     buf.writeByte(HttpConstants.LF);
-                    ByteBufUtil.copy(AsciiString.of(content), buf);
-                    buf.writeByte(HttpConstants.CR);
+                    if (randomBoolean()) {
+                        ByteBufUtil.copy(AsciiString.of("Host: localhost"), buf);
+                        buf.writeByte(HttpConstants.LF);
+                    }
+                    if (randomBoolean()) {
+                        ByteBufUtil.copy(AsciiString.of("Accept: */*"), buf);
+                        buf.writeByte(HttpConstants.LF);
+                    }
+                    if (randomBoolean()) {
+                        ByteBufUtil.copy(AsciiString.of("Content-Encoding: gzip"), buf);
+                        buf.writeByte(HttpConstants.LF);
+                    }
+                    if (randomBoolean()) {
+                        ByteBufUtil.copy(
+                            AsciiString.of("Content-Type: " + randomFrom("text/plain; charset=utf-8", "application/json; charset=utf-8")),
+                            buf
+                        );
+                        buf.writeByte(HttpConstants.LF);
+                    }
+                    // do not write a "Content-Length" header to make the request "variable length"
+                    if (randomBoolean()) {
+                        ByteBufUtil.copy(AsciiString.of("Transfer-Encoding: " + randomFrom("chunked", "gzip, chunked")), buf);
+                    } else {
+                        ByteBufUtil.copy(AsciiString.of("Transfer-Encoding: chunked"), buf);
+                    }
                     buf.writeByte(HttpConstants.LF);
-                }
-                testThreadPool.generic().submit(() -> {
+                    buf.writeByte(HttpConstants.LF);
+                    // maybe append some chunks as well
+                    String[] contentParts = randomArray(0, 4, String[]::new, () -> randomAlphaOfLengthBetween(1, 64));
+                    for (String content : contentParts) {
+                        ByteBufUtil.copy(AsciiString.of(Integer.toHexString(content.length())), buf);
+                        buf.writeByte(HttpConstants.CR);
+                        buf.writeByte(HttpConstants.LF);
+                        ByteBufUtil.copy(AsciiString.of(content), buf);
+                        buf.writeByte(HttpConstants.CR);
+                        buf.writeByte(HttpConstants.LF);
+                    }
                     ch.writeInbound(buf);
                     ch.flushInbound();
-                }).get();
-                // append some more chunks as well
-                ByteBuf buf2 = ch.alloc().buffer();
-                contentParts = randomArray(1, 4, String[]::new, () -> randomAlphaOfLengthBetween(1, 64));
-                for (String content : contentParts) {
-                    ByteBufUtil.copy(AsciiString.of(Integer.toHexString(content.length())), buf2);
+                    ByteBuf buf2 = ch.alloc().buffer();
+                    contentParts = randomArray(1, 4, String[]::new, () -> randomAlphaOfLengthBetween(1, 64));
+                    for (String content : contentParts) {
+                        ByteBufUtil.copy(AsciiString.of(Integer.toHexString(content.length())), buf2);
+                        buf2.writeByte(HttpConstants.CR);
+                        buf2.writeByte(HttpConstants.LF);
+                        ByteBufUtil.copy(AsciiString.of(content), buf2);
+                        buf2.writeByte(HttpConstants.CR);
+                        buf2.writeByte(HttpConstants.LF);
+                    }
+                    // finish chunked request
+                    ByteBufUtil.copy(AsciiString.of("0"), buf2);
                     buf2.writeByte(HttpConstants.CR);
                     buf2.writeByte(HttpConstants.LF);
-                    ByteBufUtil.copy(AsciiString.of(content), buf2);
                     buf2.writeByte(HttpConstants.CR);
                     buf2.writeByte(HttpConstants.LF);
-                }
-                // finish chunked request
-                ByteBufUtil.copy(AsciiString.of("0"), buf2);
-                buf2.writeByte(HttpConstants.CR);
-                buf2.writeByte(HttpConstants.LF);
-                buf2.writeByte(HttpConstants.CR);
-                buf2.writeByte(HttpConstants.LF);
-                testThreadPool.generic().submit(() -> {
                     ch.writeInbound(buf2);
                     ch.flushInbound();
-                }).get();
-                ch.runPendingTasks();
-                Throwable badRequestCause = badRequestCauseReference.get();
-                assertThat(badRequestCause, instanceOf(HttpHeadersValidationException.class));
-                assertThat(badRequestCause.getCause(), instanceOf(ElasticsearchException.class));
-                assertThat(((ElasticsearchException) badRequestCause.getCause()).status(), is(RestStatus.BAD_REQUEST));
-                assertThat(
-                    ((ElasticsearchException) badRequestCause.getCause()).getDetailedMessage(),
-                    containsString("OPTIONS requests with a payload body are not supported")
-                );
-            }
+                    ch.runPendingTasks();
+                    Throwable badRequestCause = badRequestCauseReference.get();
+                    assertThat(badRequestCause, instanceOf(HttpHeadersValidationException.class));
+                    assertThat(badRequestCause.getCause(), instanceOf(ElasticsearchException.class));
+                    assertThat(((ElasticsearchException) badRequestCause.getCause()).status(), is(RestStatus.BAD_REQUEST));
+                    assertThat(
+                        ((ElasticsearchException) badRequestCause.getCause()).getDetailedMessage(),
+                        containsString("OPTIONS requests with a payload body are not supported")
+                    );
+                }
+            }));
         } finally {
             testThreadPool.shutdownNow();
         }

From 56dd5387508c4967d651d1b0c0ee94886a5e68ec Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Thu, 30 May 2024 13:44:47 +0100
Subject: [PATCH 059/208] Share NetworkService instance across transport tests
 (#109202)

No need for a new instance each time.
---
 .../transport/netty4/SimpleNetty4TransportTests.java           | 3 +--
 .../transport/AbstractSimpleTransportTestCase.java             | 3 +++
 .../netty4/SimpleSecurityNetty4ServerTransportTests.java       | 1 -
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
index c2727f206a07c..6eaddf51c02b4 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.node.VersionInformation;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -63,7 +62,7 @@ protected Transport build(Settings settings, TransportVersion version, ClusterSe
             settings,
             version,
             threadPool,
-            new NetworkService(Collections.emptyList()),
+            networkService,
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
             namedWriteableRegistry,
             new NoneCircuitBreakerService(),
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index d966a21a56b5f..6ced86156d008 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.network.CloseableChannel;
 import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -152,6 +153,8 @@ protected Set<Setting<?>> getSupportedSettings() {
         return ClusterSettings.BUILT_IN_CLUSTER_SETTINGS;
     }
 
+    protected static final NetworkService networkService = new NetworkService(List.of());
+
     @Override
     @Before
     public void setUp() throws Exception {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java
index 9ff23e5e7b9d8..74b02c1d63bbf 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java
@@ -115,7 +115,6 @@ public class SimpleSecurityNetty4ServerTransportTests extends AbstractSimpleTran
     @Override
     protected Transport build(Settings settings, TransportVersion version, ClusterSettings clusterSettings, boolean doHandshake) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
-        NetworkService networkService = new NetworkService(Collections.emptyList());
         Settings settings1 = Settings.builder().put(settings).put("xpack.security.transport.ssl.enabled", true).build();
         return new TestSecurityNetty4ServerTransport(
             settings1,

From d26be67b6f3c12548e718cee82da6cbed764c872 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Thu, 30 May 2024 13:44:59 +0100
Subject: [PATCH 060/208] Add reference docs link to InboundHandler warning
 (#109203)

Seems like it would be useful to link to the relevant docs here.
---
 .../elasticsearch/common/ReferenceDocs.java   |  1 +
 .../transport/InboundHandler.java             | 15 ++++-----
 .../common/reference-docs-links.json          |  3 +-
 .../transport/InboundHandlerTests.java        | 32 +++++++++++--------
 .../org/elasticsearch/test/ESTestCase.java    |  4 +++
 5 files changed, 32 insertions(+), 23 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
index 8e370158d166a..2cac6ddb159bc 100644
--- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
+++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
@@ -72,6 +72,7 @@ public enum ReferenceDocs {
     CONTACT_SUPPORT,
     UNASSIGNED_SHARDS,
     EXECUTABLE_JNA_TMPDIR,
+    NETWORK_THREADING_MODEL,
     // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner
     ;
 
diff --git a/server/src/main/java/org/elasticsearch/transport/InboundHandler.java b/server/src/main/java/org/elasticsearch/transport/InboundHandler.java
index 79b6daef671af..37cb8931d8cb0 100644
--- a/server/src/main/java/org/elasticsearch/transport/InboundHandler.java
+++ b/server/src/main/java/org/elasticsearch/transport/InboundHandler.java
@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -169,15 +170,13 @@ private TransportResponseHandler<?> findResponseHandler(Header header) {
 
     private static void logSlowMessage(InboundMessage message, long took, long logThreshold, TransportResponseHandler<?> responseHandler) {
         if (message.getHeader().isRequest()) {
-            logger.warn("handling request [{}] took [{}ms] which is above the warn threshold of [{}ms]", message, took, logThreshold);
+            logger.warn("""
+                handling request [{}] took [{}ms] which is above the warn threshold of [{}ms]; \
+                for more information, see {}""", message, took, logThreshold, ReferenceDocs.NETWORK_THREADING_MODEL);
         } else {
-            logger.warn(
-                "handling response [{}] on handler [{}] took [{}ms] which is above the warn threshold of [{}ms]",
-                message,
-                responseHandler,
-                took,
-                logThreshold
-            );
+            logger.warn("""
+                handling response [{}] on handler [{}] took [{}ms] which is above the warn threshold of [{}ms]; \
+                for more information, see {}""", message, responseHandler, took, logThreshold, ReferenceDocs.NETWORK_THREADING_MODEL);
         }
     }
 
diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
index 503f02b25eb8d..f3e5bd7a375f1 100644
--- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
+++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
@@ -32,5 +32,6 @@
   "BOOTSTRAP_CHECK_SECURITY_MINIMAL_SETUP": "security-minimal-setup.html",
   "CONTACT_SUPPORT": "troubleshooting.html#troubleshooting-contact-support",
   "UNASSIGNED_SHARDS": "red-yellow-cluster-status.html",
-  "EXECUTABLE_JNA_TMPDIR": "executable-jna-tmpdir.html"
+  "EXECUTABLE_JNA_TMPDIR": "executable-jna-tmpdir.html",
+  "NETWORK_THREADING_MODEL": "modules-network.html#modules-network-threading-model"
 }
diff --git a/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java b/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java
index 23f9a7367298f..c9e1a7dbc0cfd 100644
--- a/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java
@@ -272,7 +272,12 @@ public void testLogsSlowInboundProcessing() throws Exception {
             final TransportVersion remoteVersion = TransportVersion.current();
 
             mockLog.addExpectation(
-                new MockLog.SeenEventExpectation("expected slow request", EXPECTED_LOGGER_NAME, Level.WARN, "handling request ")
+                new MockLog.SeenEventExpectation(
+                    "expected slow request",
+                    EXPECTED_LOGGER_NAME,
+                    Level.WARN,
+                    "handling request*modules-network.html#modules-network-threading-model"
+                )
             );
 
             final long requestId = randomNonNegativeLong();
@@ -285,13 +290,11 @@ public void testLogsSlowInboundProcessing() throws Exception {
             BytesStreamOutput byteData = new BytesStreamOutput();
             TaskId.EMPTY_TASK_ID.writeTo(byteData);
             TransportVersion.writeVersion(remoteVersion, byteData);
-            final InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.wrap(byteData.bytes()), () -> {
-                try {
-                    TimeUnit.SECONDS.sleep(1L);
-                } catch (InterruptedException e) {
-                    throw new AssertionError(e);
-                }
-            });
+            final InboundMessage requestMessage = new InboundMessage(
+                requestHeader,
+                ReleasableBytesReference.wrap(byteData.bytes()),
+                () -> safeSleep(TimeValue.timeValueSeconds(1))
+            );
             requestHeader.actionName = TransportHandshaker.HANDSHAKE_ACTION_NAME;
             requestHeader.headers = Tuple.tuple(Map.of(), Map.of());
             handler.inboundMessage(channel, requestMessage);
@@ -299,7 +302,12 @@ public void testLogsSlowInboundProcessing() throws Exception {
             mockLog.assertAllExpectationsMatched();
 
             mockLog.addExpectation(
-                new MockLog.SeenEventExpectation("expected slow response", EXPECTED_LOGGER_NAME, Level.WARN, "handling response ")
+                new MockLog.SeenEventExpectation(
+                    "expected slow response",
+                    EXPECTED_LOGGER_NAME,
+                    Level.WARN,
+                    "handling response*modules-network.html#modules-network-threading-model"
+                )
             );
 
             final long responseId = randomNonNegativeLong();
@@ -310,11 +318,7 @@ public void testLogsSlowInboundProcessing() throws Exception {
                 @SuppressWarnings("rawtypes")
                 public void onResponseReceived(long requestId, Transport.ResponseContext context) {
                     assertEquals(responseId, requestId);
-                    try {
-                        TimeUnit.SECONDS.sleep(1L);
-                    } catch (InterruptedException e) {
-                        throw new AssertionError(e);
-                    }
+                    safeSleep(TimeValue.timeValueSeconds(1));
                 }
             });
             handler.inboundMessage(channel, new InboundMessage(responseHeader, ReleasableBytesReference.empty(), () -> {}));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 14269a8835f57..02be27b92a1eb 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -2187,6 +2187,10 @@ public static <T> T safeGet(Future<T> future) {
         }
     }
 
+    public static void safeSleep(TimeValue timeValue) {
+        safeSleep(timeValue.millis());
+    }
+
     public static void safeSleep(long millis) {
         try {
             Thread.sleep(millis);

From 0ce54d7e20a4cf2681f5fa9c385cfa791fc9489e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= <ivancea96@outlook.com>
Date: Thu, 30 May 2024 15:28:40 +0200
Subject: [PATCH 061/208] ESQL: Change "substring" function to not return null
 on empty string (#109174)

Currently, `substring("", 1, 1)` returns null instead of an empty string. This PR changes the behaviour to the expected one, as in other languages and databases.

Unrelated to this PR's intent, returning null is causing the function to fail with a NPE, so this is currently raising an error, and doesn't affect existing queries.

Related issue: https://github.com/elastic/elasticsearch/issues/109095
---
 docs/changelog/109174.yaml                     |  5 +++++
 .../elasticsearch/xpack/esql/CsvTestUtils.java |  3 ++-
 .../src/main/resources/string.csv-spec         |  8 ++++++++
 .../xpack/esql/action/EsqlCapabilities.java    |  6 ++++++
 .../function/scalar/string/Substring.java      |  6 +++---
 .../function/scalar/string/SubstringTests.java | 18 ++++++++++++++++++
 6 files changed, 42 insertions(+), 4 deletions(-)
 create mode 100644 docs/changelog/109174.yaml

diff --git a/docs/changelog/109174.yaml b/docs/changelog/109174.yaml
new file mode 100644
index 0000000000000..5cd57ebd34ac6
--- /dev/null
+++ b/docs/changelog/109174.yaml
@@ -0,0 +1,5 @@
+pr: 109174
+summary: "ESQL: Change \"substring\" function to not return null on empty string"
+area: ES|QL
+type: bug
+issues: []
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
index 7d1c168bd203f..d78905cc2cffa 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java
@@ -354,7 +354,8 @@ public static ExpectedResults loadCsvSpecValues(String csv) {
                 for (int i = 0; i < row.size(); i++) {
                     String value = row.get(i);
                     if (value == null) {
-                        rowValues.add(null);
+                        // Empty cells are converted to null by SuperCSV. We convert them back to empty strings.
+                        rowValues.add("");
                         continue;
                     }
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
index 6322746318230..13616e5146949 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
@@ -195,6 +195,14 @@ emp_no:integer | last_name:keyword | x:keyword | z:keyword
 10010 | Piveteau  | P | a
 ;
 
+substring empty string
+required_capability: fn_substring_empty_null
+row sub = substring("", 1, 3);
+
+sub:keyword
+""
+;
+
 substring Emoji#[skip:-8.13.99,reason:bug fix in 8.14]
 row a = "🐱Meow!🐶Woof!" | eval sub1 = substring(a, 2) | eval sub2 = substring(a, 2, 100);
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index e8f136c297ce0..675b99c61bfbe 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -32,6 +32,11 @@ public class EsqlCapabilities {
      */
     private static final String FN_IP_PREFIX = "fn_ip_prefix";
 
+    /**
+     * Fix on function {@code SUBSTRING} that makes it not return null on empty strings.
+     */
+    private static final String FN_SUBSTRING_EMPTY_NULL = "fn_substring_empty_null";
+
     /**
      * Optimization for ST_CENTROID changed some results in cartesian data. #108713
      */
@@ -53,6 +58,7 @@ private static Set<String> capabilities() {
         List<String> caps = new ArrayList<>();
         caps.add(FN_CBRT);
         caps.add(FN_IP_PREFIX);
+        caps.add(FN_SUBSTRING_EMPTY_NULL);
         caps.add(ST_CENTROID_AGG_OPTIMIZED);
         caps.add(METADATA_IGNORED_FIELD);
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java
index 5c84d5bcee853..4f3d0e15eef72 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java
@@ -111,12 +111,12 @@ static BytesRef process(BytesRef str, int start) {
 
     @Evaluator
     static BytesRef process(BytesRef str, int start, int length) {
-        if (str.length == 0) {
-            return null;
-        }
         if (length < 0) {
             throw new IllegalArgumentException("Length parameter cannot be negative, found [" + length + "]");
         }
+        if (str.length == 0) {
+            return str;
+        }
         int codePointCount = UnicodeUtil.codePointCount(str);
         int indexStart = indexStart(codePointCount, start);
         int indexEnd = Math.min(codePointCount, indexStart + length);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
index 36ace66290fc1..c2df6f61c1d25 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
@@ -78,6 +78,24 @@ public static Iterable<Object[]> parameters() {
                                     equalTo(new BytesRef(text.substring(start - 1, start + length - 1)))
                                 );
                             }
+                        ),
+                        new TestCaseSupplier(
+                            "Substring empty string",
+                            List.of(DataTypes.TEXT, DataTypes.INTEGER, DataTypes.INTEGER),
+                            () -> {
+                                int start = between(1, 8);
+                                int length = between(1, 10 - start);
+                                return new TestCaseSupplier.TestCase(
+                                    List.of(
+                                        new TestCaseSupplier.TypedData(new BytesRef(""), DataTypes.TEXT, "str"),
+                                        new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
+                                        new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "end")
+                                    ),
+                                    "SubstringEvaluator[str=Attribute[channel=0], start=Attribute[channel=1], length=Attribute[channel=2]]",
+                                    DataTypes.KEYWORD,
+                                    equalTo(new BytesRef(""))
+                                );
+                            }
                         )
                     )
                 )

From 489aac90e05c49dd3a643639cff4de0b1ae95e77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= <ivancea96@outlook.com>
Date: Thu, 30 May 2024 15:29:36 +0200
Subject: [PATCH 062/208] ESQL: Fix IpPrefix function not handling correctly
 ByteRefs (#109205)

- Fix IpPrefix function not using ByteRef offset on calculations, leading to wrong bytes being written to the prefix.
- Fixed the CSVTest case, which was also wrong, leading to a "_wrong * wrong = correct_" situation.
-  ~~Added` a BytesRef randomizer and applied it to the IpPrefixTests suppliers~~
    - ~~This highlighted another bug, which was fixed too~~
    - (Moved to https://github.com/elastic/elasticsearch/pull/109207)

Fixes https://github.com/elastic/elasticsearch/issues/109198
---
 docs/changelog/109205.yaml                            |  6 ++++++
 muted-tests.yml                                       |  3 ---
 .../qa/testFixtures/src/main/resources/ip.csv-spec    |  2 +-
 .../esql/expression/function/scalar/ip/IpPrefix.java  | 11 +++++++++--
 4 files changed, 16 insertions(+), 6 deletions(-)
 create mode 100644 docs/changelog/109205.yaml

diff --git a/docs/changelog/109205.yaml b/docs/changelog/109205.yaml
new file mode 100644
index 0000000000000..10f13a6549fbc
--- /dev/null
+++ b/docs/changelog/109205.yaml
@@ -0,0 +1,6 @@
+pr: 109205
+summary: "ESQL: Fix `IpPrefix` function not handling correctly `ByteRefs`"
+area: ES|QL
+type: bug
+issues:
+ - 109198
diff --git a/muted-tests.yml b/muted-tests.yml
index 262b1509ca58e..b64f490524c86 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -49,9 +49,6 @@ tests:
 - class: "org.elasticsearch.upgrades.SearchStatesIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109190"
   method: "testBWCSearchStates"
-- class: "org.elasticsearch.xpack.esql.CsvTests"
-  issue: "https://github.com/elastic/elasticsearch/issues/109198"
-  method: "test {ip.IpPrefixLengthFromColumn}"
 - class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109200"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
index 0de64d3e2d9d4..61f529d60bf90 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
@@ -531,7 +531,7 @@ required_capability: fn_ip_prefix
 from hosts
 | where host == "alpha"
 | sort card
-| eval prefix = ip_prefix(ip0, 24, 128)
+| eval prefix = ip_prefix(ip0, 24, 120)
 | keep card, ip0, prefix;
 
 card:keyword | ip0:ip    | prefix:ip
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
index 5c576101668b3..9d429a620ed6e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
@@ -136,7 +136,14 @@ static BytesRef process(
             throw new IllegalArgumentException("Prefix length v6 must be in range [0, 128], found " + prefixLengthV6);
         }
 
-        boolean isIpv4 = Arrays.compareUnsigned(ip.bytes, 0, IPV4_PREFIX.length, IPV4_PREFIX, 0, IPV4_PREFIX.length) == 0;
+        boolean isIpv4 = Arrays.compareUnsigned(
+            ip.bytes,
+            ip.offset,
+            ip.offset + IPV4_PREFIX.length,
+            IPV4_PREFIX,
+            0,
+            IPV4_PREFIX.length
+        ) == 0;
 
         if (isIpv4) {
             makePrefix(ip, scratch, 12 + prefixLengthV4 / 8, prefixLengthV4 % 8);
@@ -154,7 +161,7 @@ private static void makePrefix(BytesRef ip, BytesRef scratch, int fullBytes, int
         // Copy the last byte ignoring the trailing bits
         if (remainingBits > 0) {
             byte lastByteMask = (byte) (0xFF << (8 - remainingBits));
-            scratch.bytes[fullBytes] = (byte) (ip.bytes[fullBytes] & lastByteMask);
+            scratch.bytes[fullBytes] = (byte) (ip.bytes[ip.offset + fullBytes] & lastByteMask);
         }
 
         // Copy the last empty bytes

From d367c117937865bdbe9a06b7638b1ba7f308be5f Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Thu, 30 May 2024 17:00:15 +0200
Subject: [PATCH 063/208] Do not execute listeners when progress is updated to
 the end of the range (#108095)

ProgressListenableActionFuture executes listeners once the
progress is updated to a value the listener is waiting for.

But when a listener waits for the exact end of a range/gap,
there is no need to execute it at the time the progress is
updated to the end of the range. Instead we can leave the
listener and it will be executed when the range/gap is
completed (which should happen just after).

I'd like to propose this change because we can have read
listeners that use CacheFile#tryRead which relies on
SparseFileTracker#checkAvailable and the volatile
SparseFileTracker#complete field, which is only updated
when the range is completed, just before executing listeners.

If listeners are executed when the progress is updated to
the end of the range, they may call tryRead before the
SparseFileTracker#complete field is updated, making the
fast read path fails.
---
 .../ProgressListenableActionFuture.java       | 29 +++++++++++--------
 .../common/SparseFileTrackerTests.java        | 18 +++++-------
 2 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ProgressListenableActionFuture.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ProgressListenableActionFuture.java
index 0912da200735e..6381f1c1e211d 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ProgressListenableActionFuture.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ProgressListenableActionFuture.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.core.Tuple;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -25,11 +24,13 @@
  */
 class ProgressListenableActionFuture extends PlainActionFuture<Long> {
 
+    private record PositionAndListener(long position, ActionListener<Long> listener) {}
+
     protected final long start;
     protected final long end;
 
     // modified under 'this' mutex
-    private volatile List<Tuple<Long, ActionListener<Long>>> listeners;
+    private volatile List<PositionAndListener> listeners;
     protected volatile long progress;
     private volatile boolean completed;
 
@@ -55,7 +56,7 @@ private boolean invariant() {
             assert completed == false || listeners == null;
             assert start <= progress : start + " <= " + progress;
             assert progress <= end : progress + " <= " + end;
-            assert listeners == null || listeners.stream().allMatch(listener -> progress < listener.v1());
+            assert listeners == null || listeners.stream().allMatch(listener -> progress < listener.position());
         }
         return true;
     }
@@ -78,17 +79,20 @@ public void onProgress(final long progressValue) {
             assert false : end + " < " + progressValue;
             throw new IllegalArgumentException("Cannot update progress with a value greater than [end=" + end + ']');
         }
+        if (progressValue == end) {
+            return; // reached the end of the range, listeners will be completed by {@link #onResponse(Long)}
+        }
 
         List<ActionListener<Long>> listenersToExecute = null;
         synchronized (this) {
             assert this.progress < progressValue : this.progress + " < " + progressValue;
             this.progress = progressValue;
 
-            final List<Tuple<Long, ActionListener<Long>>> listenersCopy = this.listeners;
+            final List<PositionAndListener> listenersCopy = this.listeners;
             if (listenersCopy != null) {
-                List<Tuple<Long, ActionListener<Long>>> listenersToKeep = null;
-                for (Tuple<Long, ActionListener<Long>> listener : listenersCopy) {
-                    if (progressValue < listener.v1()) {
+                List<PositionAndListener> listenersToKeep = null;
+                for (PositionAndListener listener : listenersCopy) {
+                    if (progressValue < listener.position()) {
                         if (listenersToKeep == null) {
                             listenersToKeep = new ArrayList<>();
                         }
@@ -97,7 +101,7 @@ public void onProgress(final long progressValue) {
                         if (listenersToExecute == null) {
                             listenersToExecute = new ArrayList<>();
                         }
-                        listenersToExecute.add(listener.v2());
+                        listenersToExecute.add(listener.listener());
                     }
                 }
                 this.listeners = listenersToKeep;
@@ -134,7 +138,8 @@ private void ensureNotCompleted() {
     @Override
     protected void done(boolean success) {
         super.done(success);
-        final List<Tuple<Long, ActionListener<Long>>> listenersToExecute;
+        final List<PositionAndListener> listenersToExecute;
+        assert invariant();
         synchronized (this) {
             assert completed == false;
             completed = true;
@@ -142,7 +147,7 @@ protected void done(boolean success) {
             listeners = null;
         }
         if (listenersToExecute != null) {
-            listenersToExecute.stream().map(Tuple::v2).forEach(listener -> executeListener(listener, this::actionResult));
+            listenersToExecute.forEach(listener -> executeListener(listener.listener(), this::actionResult));
         }
         assert invariant();
     }
@@ -162,11 +167,11 @@ public void addListener(ActionListener<Long> listener, long value) {
             if (completed || value <= progressValue) {
                 executeImmediate = true;
             } else {
-                List<Tuple<Long, ActionListener<Long>>> listenersCopy = this.listeners;
+                List<PositionAndListener> listenersCopy = this.listeners;
                 if (listenersCopy == null) {
                     listenersCopy = new ArrayList<>();
                 }
-                listenersCopy.add(Tuple.tuple(value, listener));
+                listenersCopy.add(new PositionAndListener(value, listener));
                 this.listeners = listenersCopy;
             }
         }
diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
index 41b323b769a93..6f1635e67ca4b 100644
--- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
@@ -119,7 +119,7 @@ public void testInvalidRange() {
         }
     }
 
-    public void testCallsListenerWhenWholeRangeIsAvailable() {
+    public void testCallsListenerWhenRangeIsCompleted() {
         final byte[] fileContents = new byte[between(0, 1000)];
         final SparseFileTracker sparseFileTracker = new SparseFileTracker("test", fileContents.length);
 
@@ -155,21 +155,19 @@ public void testCallsListenerWhenWholeRangeIsAvailable() {
                 final SparseFileTracker.Gap gap = gaps.get(gapIndex);
                 assertThat(gap.start(), greaterThanOrEqualTo(start));
                 assertThat(gap.end(), lessThanOrEqualTo(end));
-                // listener is notified when the last gap is completed
-                final AtomicBoolean shouldNotifyListener = new AtomicBoolean();
                 for (long i = gap.start(); i < gap.end(); i++) {
                     assertThat(fileContents[toIntBytes(i)], equalTo(UNAVAILABLE));
                     fileContents[toIntBytes(i)] = AVAILABLE;
-                    // listener is notified when the progress reached the last byte of the last gap
-                    if ((gapIndex == gaps.size() - 1) && (i == gap.end() - 1L)) {
-                        assertTrue(shouldNotifyListener.compareAndSet(false, true));
-                        expectNotification.set(true);
-                    }
                     gap.onProgress(i + 1L);
-                    assertThat(wasNotified.get(), equalTo(shouldNotifyListener.get()));
+                    assertThat(wasNotified.get(), equalTo(false));
+                }
+                // listener is notified when the last gap is completed
+                if (gapIndex == gaps.size() - 1) {
+                    expectNotification.set(true);
                 }
-                assertThat(wasNotified.get(), equalTo(shouldNotifyListener.get()));
+                assertThat(wasNotified.get(), equalTo(false));
                 gap.onCompletion();
+                assertThat(wasNotified.get(), equalTo(expectNotification.get()));
             }
             assertTrue(wasNotified.get());
         }

From a52134c15f407eb5aa2ef0b029fb78c6fda11d1c Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Thu, 30 May 2024 17:25:15 +0200
Subject: [PATCH 064/208] Log index scoped setting update at debug level
 (#109094)

Setting updates are logged at info level. Typically these log messages are only useful for debugging purposes. Some index scoped settings are updated more frequently then other type of settings. Typically the index settings that are updated via an automatic process (such as the index.time_series.end_time index setting). These setting updates can add to a significant volume of info log messages.

This PR changes the log level of setting update log messages for index settings only from INFO to DEBUG. This to lower the default volume of info log messages.

Closes #107473
---
 .../common/settings/Setting.java              | 14 +++++++---
 .../common/settings/SettingTests.java         |  4 +--
 .../common/settings/SettingsFilterTests.java  | 26 +++++++++++++++++++
 .../index/MergeSchedulerSettingsTests.java    |  4 +--
 4 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java
index a385950e10922..71882e0c68191 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.common.settings;
 
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
@@ -1843,11 +1844,18 @@ public void diff(Settings.Builder builder, Settings source, Settings defaultSett
     }
 
     static void logSettingUpdate(Setting<?> setting, Settings current, Settings previous, Logger logger) {
-        if (logger.isInfoEnabled()) {
+        Level level = setting.hasIndexScope() ? Level.DEBUG : Level.INFO;
+        if (logger.isEnabled(level)) {
             if (setting.isFiltered()) {
-                logger.info("updating [{}]", setting.key);
+                logger.log(level, "updating [{}]", setting.key);
             } else {
-                logger.info("updating [{}] from [{}] to [{}]", setting.key, setting.getLogString(previous), setting.getLogString(current));
+                logger.log(
+                    level,
+                    "updating [{}] from [{}] to [{}]",
+                    setting.key,
+                    setting.getLogString(previous),
+                    setting.getLogString(current)
+                );
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java
index 8c6b2692e1ca1..62f143dfa9fd4 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java
@@ -1423,7 +1423,7 @@ public void testNonSecureSettingInKeystore() {
     }
 
     @TestLogging(
-        value = "org.elasticsearch.common.settings.IndexScopedSettings:INFO",
+        value = "org.elasticsearch.common.settings.IndexScopedSettings:DEBUG",
         reason = "to ensure we log INFO-level messages from IndexScopedSettings"
     )
     public void testLogSettingUpdate() throws Exception {
@@ -1438,7 +1438,7 @@ public void testLogSettingUpdate() throws Exception {
                 new MockLog.SeenEventExpectation(
                     "message",
                     "org.elasticsearch.common.settings.IndexScopedSettings",
-                    Level.INFO,
+                    Level.DEBUG,
                     "updating [index.refresh_interval] from [20s] to [10s]"
                 ) {
                     @Override
diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
index a42e09a1c080b..10f6bc220613a 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.config.Configurator;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.rest.RestRequest;
@@ -99,6 +100,31 @@ public void testFilteredSettingIsNotLogged() throws Exception {
         );
     }
 
+    public void testIndexScopeSettingUpdateLoggedAsDebug() throws Exception {
+        Settings oldSettings = Settings.builder().put("key", "old").build();
+        Settings newSettings = Settings.builder().put("key", "new").build();
+
+        // With INFO log level nothing gets logged.
+        Setting<String> filteredSetting = Setting.simpleString("key", Property.IndexScope);
+        assertExpectedLogMessages((testLogger) -> Setting.logSettingUpdate(filteredSetting, newSettings, oldSettings, testLogger));
+
+        try {
+            // With DEBUG log level something gets logged
+            Configurator.setLevel("org.elasticsearch.test", Level.DEBUG);
+            assertExpectedLogMessages(
+                (logger) -> Setting.logSettingUpdate(filteredSetting, newSettings, oldSettings, logger),
+                new MockLog.SeenEventExpectation(
+                    "regular logging",
+                    "org.elasticsearch.test",
+                    Level.DEBUG,
+                    "updating [key] from [old] to [new]"
+                )
+            );
+        } finally {
+            Configurator.setLevel("org.elasticsearch.test", Level.INFO);
+        }
+    }
+
     public void testRegularSettingUpdateIsFullyLogged() throws Exception {
         Settings oldSettings = Settings.builder().put("key", "old").build();
         Settings newSettings = Settings.builder().put("key", "new").build();
diff --git a/server/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java b/server/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java
index 9ee78a3008f32..bd77a2b7a38a5 100644
--- a/server/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/index/MergeSchedulerSettingsTests.java
@@ -39,11 +39,11 @@ public void append(LogEvent event) {
             String message = event.getMessage().getFormattedMessage();
             if (event.getLevel() == Level.TRACE && event.getLoggerName().endsWith("lucene.iw")) {
             }
-            if (event.getLevel() == Level.INFO
+            if (event.getLevel() == Level.DEBUG
                 && message.contains("updating [index.merge.scheduler.max_thread_count] from [10000] to [1]")) {
                 sawUpdateMaxThreadCount = true;
             }
-            if (event.getLevel() == Level.INFO
+            if (event.getLevel() == Level.DEBUG
                 && message.contains("updating [index.merge.scheduler.auto_throttle] from [true] to [false]")) {
                 sawUpdateAutoThrottle = true;
             }

From aaf7c748c26dbd472584adfc93843e9a78897063 Mon Sep 17 00:00:00 2001
From: Artem Prigoda <artem.prigoda@elastic.co>
Date: Thu, 30 May 2024 18:17:29 +0200
Subject: [PATCH 065/208] Unmute #109190 (#109211)

It looks like the underlying issue has been solved by #109166

Resolves #109190
---
 muted-tests.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index b64f490524c86..303e0196782ad 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -46,9 +46,6 @@ tests:
 - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109189"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
-- class: "org.elasticsearch.upgrades.SearchStatesIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109190"
-  method: "testBWCSearchStates"
 - class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109200"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"

From 28588e084769266fb4d7f364439677b0f5b58e17 Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Thu, 30 May 2024 11:37:54 -0500
Subject: [PATCH 066/208] update release notes (#109127)

---
 docs/changelog/105709.yaml                    |  3 +--
 docs/changelog/105714.yaml                    |  2 +-
 .../reference/migration/migrate_8_14.asciidoc | 25 ++++++++++++++++++-
 3 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/docs/changelog/105709.yaml b/docs/changelog/105709.yaml
index 568d60a86334e..41b6e749d9270 100644
--- a/docs/changelog/105709.yaml
+++ b/docs/changelog/105709.yaml
@@ -1,6 +1,5 @@
 pr: 105709
-summary: Disable validate when rewrite parameter is sent and the index access control
-  list is non-null
+summary: Apply stricter Document Level Security (DLS) rules for the validate query API with the rewrite parameter.
 area: Security
 type: bug
 issues: []
diff --git a/docs/changelog/105714.yaml b/docs/changelog/105714.yaml
index 20301a4c03e83..b6ab5e128c72c 100644
--- a/docs/changelog/105714.yaml
+++ b/docs/changelog/105714.yaml
@@ -1,5 +1,5 @@
 pr: 105714
-summary: Cross check livedocs for terms aggs when index access control list is non-null
+summary: Apply stricter Document Level Security (DLS) rules for terms aggregations when min_doc_count is set to 0.
 area: "Aggregations"
 type: bug
 issues: []
diff --git a/docs/reference/migration/migrate_8_14.asciidoc b/docs/reference/migration/migrate_8_14.asciidoc
index 672c472c14325..bdff8bef246b5 100644
--- a/docs/reference/migration/migrate_8_14.asciidoc
+++ b/docs/reference/migration/migrate_8_14.asciidoc
@@ -16,5 +16,28 @@ coming::[8.14.0]
 [[breaking-changes-8.14]]
 === Breaking changes
 
-There are no breaking changes in {es} 8.14.
+The following changes in {es} 8.14 might affect your applications
+and prevent them from operating normally.
+Before upgrading to 8.14, review these changes and take the described steps
+to mitigate the impact.
 
+[discrete]
+[[breaking-changes-8.14-0]]
+
+[discrete]
+[[breaking_814_dls_changes]]
+==== Stricter Document Level Security (DLS)
+
+[[stricter_dls_814]]
+.Document Level Security (DLS) applies stricter checks for the validate query API and for terms aggregations when min_doc_count is set to 0.
+
+[%collapsible]
+====
+*Details* +
+When Document Level Security (DLS) is applied to terms aggregations and min_doc_count is set to 0, stricter security rules apply.
+
+When Document Level Security (DLS) is applied to the validate query API with the rewrite parameter, stricter security rules apply.
+
+*Impact* +
+If needed, test workflows with DLS enabled to ensure that the stricter security rules do not impact your application.
+====

From a7119a38674d32d5acc2e746d72b922b93802f3a Mon Sep 17 00:00:00 2001
From: Mikhail Berezovskiy <mikhail@elastic.co>
Date: Thu, 30 May 2024 10:59:12 -0700
Subject: [PATCH 067/208] add package-info to cluster.routing (#109140)

Add package-info to cluster routing. Does not include `allocation`
package information, but it might be better to have routing and
allocation description in one place.
---
 .../cluster/routing/package-info.java         | 85 +++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/routing/package-info.java

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/package-info.java b/server/src/main/java/org/elasticsearch/cluster/routing/package-info.java
new file mode 100644
index 0000000000000..6a4ed5861d94b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/package-info.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * <p>The cluster routing package provides routing information and manages shard
+ * allocations in a cluster. The routing part contains two different views of
+ * shards in {@link org.elasticsearch.cluster.routing.RoutingTable} and
+ * {@link org.elasticsearch.cluster.routing.RoutingNodes}. RoutingTable provides
+ * a view from index to shard to node. RoutingNodes provides view from node to
+ * shard. Shard allocation is a process of assigning and moving shards between
+ * nodes. For more details about allocation see
+ * {@link org.elasticsearch.cluster.routing.allocation}.</p>
+ *
+ * <b>Routing Table</b>
+ *
+ * <p>The routing table provide a view of global cluster state from an index
+ * perspective. It's a mapping from indices to shards and shards to nodes, where
+ * the relationship between shard and node may not exist if a shard
+ * allocation is not possible.
+ *
+ * <p>{@link org.elasticsearch.cluster.routing.RoutingTable} is the access
+ * point to the routing information. RoutingTable is a part of the
+ * {@link org.elasticsearch.cluster.ClusterState}. It maps index names to
+ * {@link org.elasticsearch.cluster.routing.IndexRoutingTable}.
+ * {@link org.elasticsearch.cluster.routing.IndexRoutingTable}, in turn,
+ * holds a list of shards in that index,
+ * {@link org.elasticsearch.cluster.routing.IndexShardRoutingTable}. Each
+ * shard has one or more instances: a primary and replicas. An
+ * IndexShardRoutingTable contains information about all shard instances for
+ * a specific shard, {@link org.elasticsearch.cluster.routing.ShardRouting}.
+ * The ShardRouting is the state of a shard instance in a cluster. There are
+ * several states of ShardRouting: unassigned, initializing, relocating,
+ * started.</p>
+ *
+ * An example of routing table:
+ *
+ * <pre>
+ * {@code
+ * └── ClusterState
+ *     └── RoutingTable
+ *         ├── index1-IndexRoutingTable
+ *         │   ├── shard1-IndexShardRoutingTable
+ *         │   │   ├── primary-ShardRouting
+ *         │   │   │   └── STARTED-node1
+ *         │   │   ├── replica1-ShardRouting
+ *         │   │   │   └── INITIALIZING-node2
+ *         │   │   └── replica2-ShardRouting
+ *         │   │       └── UNASSIGNED
+ *         │   └── shard2-IndexShardRoutingTable ...
+ *         └── index2-IndexRoutingTable ...
+ * }
+ * </pre>
+ *
+ *
+ * <b>Routing Nodes</b>
+ *
+ * <p>{@link org.elasticsearch.cluster.routing.RoutingNode} provides a view
+ * from node to shard routing. There is a RoutingNode for every
+ * {@link org.elasticsearch.cluster.node.DiscoveryNode}. It contains
+ * information about  all shards and their state on this node.
+ * {@link org.elasticsearch.cluster.routing.RoutingNodes} (plural) provide
+ * aggregated view from all cluster nodes. It supports finding all shards by
+ * specific node or finding all nodes for specific shard.</p>
+ *
+ * <b>Reroute</b>
+ *
+ * <p>Reroute is a process of routing update in the cluster. Routing update
+ * may start allocation process, which assign or move shards around. When
+ * cluster has an update that impacts routing (for example: node join
+ * cluster, index created, snapshot restored, ...) the code that handles
+ * cluster update should trigger reroute. There are 2 ways to trigger reroute -
+ * batched with
+ * {@link org.elasticsearch.cluster.routing.BatchedRerouteService} and
+ * unbatched
+ * {@link org.elasticsearch.cluster.routing.allocation.AllocationService}.
+ * Batched reroute can combine multiple requests into one (used when starting
+ * initializing shards). Unbatched reroute allows to mix other cluster state
+ * changes that might be required to create or delete index.</p>
+ */
+package org.elasticsearch.cluster.routing;

From ee0ec9fa08db880be5946f2993176f755b276e00 Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Thu, 30 May 2024 14:25:33 -0700
Subject: [PATCH 068/208] Fix TSDBRestEsqlIT (#109223)

---
 .../esql/qa/single_node/TSDBRestEsqlIT.java   | 36 ++++++++++---------
 ...sdb-mapping.json => tsdb-k8s-mapping.json} |  0
 2 files changed, 19 insertions(+), 17 deletions(-)
 rename x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/resources/{tsdb-mapping.json => tsdb-k8s-mapping.json} (100%)

diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java
index f3052f4e30b9e..543630292c3e4 100644
--- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java
+++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java
@@ -9,7 +9,6 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 
 import org.apache.http.util.EntityUtils;
-import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.Build;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
@@ -33,7 +32,6 @@
  * This while the functionality is gated behind a query pragma.
  */
 @ThreadLeakFilters(filters = TestClustersThreadFilter.class)
-@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107557")
 public class TSDBRestEsqlIT extends ESRestTestCase {
     @ClassRule
     public static ElasticsearchCluster cluster = Clusters.testCluster();
@@ -48,7 +46,7 @@ public void testTimeSeriesQuerying() throws IOException {
         var settings = Settings.builder()
             .loadFromStream("tsdb-settings.json", TSDBRestEsqlIT.class.getResourceAsStream("/tsdb-settings.json"), false)
             .build();
-        String mapping = CsvTestsDataLoader.readTextFile(TSDBRestEsqlIT.class.getResource("/tsdb-mapping.json"));
+        String mapping = CsvTestsDataLoader.readTextFile(TSDBRestEsqlIT.class.getResource("/tsdb-k8s-mapping.json"));
         createIndex("k8s", settings, mapping);
 
         Request bulk = new Request("POST", "/k8s/_bulk");
@@ -64,7 +62,7 @@ public void testTimeSeriesQuerying() throws IOException {
         assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
 
         RestEsqlTestCase.RequestObjectBuilder builder = RestEsqlTestCase.requestObjectBuilder()
-            .query("FROM k8s | KEEP k8s.pod.name, @timestamp");
+            .query("FROM k8s | KEEP k8s.pod.name, @timestamp | SORT @timestamp, k8s.pod.name");
         builder.pragmas(Settings.builder().put("time_series", true).build());
         Map<String, Object> result = runEsqlSync(builder);
         @SuppressWarnings("unchecked")
@@ -77,24 +75,28 @@ public void testTimeSeriesQuerying() throws IOException {
         @SuppressWarnings("unchecked")
         List<List<?>> values = (List<List<?>>) result.get("values");
         assertEquals(8, values.size());
-        assertEquals("hamster", values.get(0).get(0));
-        assertEquals("2021-04-29T17:29:22.470Z", values.get(0).get(1));
-        assertEquals("hamster", values.get(1).get(0));
-        assertEquals("2021-04-29T17:29:12.470Z", values.get(1).get(1));
+        assertEquals("2021-04-29T17:29:12.470Z", values.get(0).get(1));
+        assertEquals("cat", values.get(0).get(0));
 
-        assertEquals("rat", values.get(2).get(0));
-        assertEquals("2021-04-29T17:29:22.470Z", values.get(2).get(1));
+        assertEquals("2021-04-29T17:29:12.470Z", values.get(0).get(1));
+        assertEquals("cow", values.get(1).get(0));
+
+        assertEquals("2021-04-29T17:29:12.470Z", values.get(0).get(1));
+        assertEquals("hamster", values.get(2).get(0));
+
+        assertEquals("2021-04-29T17:29:12.470Z", values.get(0).get(1));
         assertEquals("rat", values.get(3).get(0));
-        assertEquals("2021-04-29T17:29:12.470Z", values.get(3).get(1));
 
-        assertEquals("cow", values.get(4).get(0));
+        assertEquals("2021-04-29T17:29:22.470Z", values.get(4).get(1));
+        assertEquals("cat", values.get(4).get(0));
+
         assertEquals("2021-04-29T17:29:22.470Z", values.get(4).get(1));
         assertEquals("cow", values.get(5).get(0));
-        assertEquals("2021-04-29T17:29:12.470Z", values.get(5).get(1));
 
-        assertEquals("cat", values.get(6).get(0));
-        assertEquals("2021-04-29T17:29:22.470Z", values.get(6).get(1));
-        assertEquals("cat", values.get(7).get(0));
-        assertEquals("2021-04-29T17:29:12.470Z", values.get(7).get(1));
+        assertEquals("2021-04-29T17:29:22.470Z", values.get(4).get(1));
+        assertEquals("hamster", values.get(6).get(0));
+
+        assertEquals("2021-04-29T17:29:22.470Z", values.get(4).get(1));
+        assertEquals("rat", values.get(7).get(0));
     }
 }
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/resources/tsdb-mapping.json b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/resources/tsdb-k8s-mapping.json
similarity index 100%
rename from x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/resources/tsdb-mapping.json
rename to x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/resources/tsdb-k8s-mapping.json

From 80c32951c086ac2c04f394c0d01ef13c1840e9fc Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Thu, 30 May 2024 18:17:54 -0400
Subject: [PATCH 069/208] [ESQL] Remove unneeded OptionalArgument specification
 on Pow function (#108872)

Removes the `OptionalArgument` interface from `Pow`.  Pow has no
optional arguments, and including this interface was a copy/paste error
most likely.
---
 .../xpack/esql/expression/function/scalar/math/Pow.java        | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java
index dab7de2b455ce..0e60e25af1981 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.compute.ann.Evaluator;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.function.OptionalArgument;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -29,7 +28,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric;
 
-public class Pow extends EsqlScalarFunction implements OptionalArgument {
+public class Pow extends EsqlScalarFunction {
 
     private final Expression base;
     private final Expression exponent;

From 84ca3be9c688c81d9d1f2fca23b4c81872be5a27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Slobodan=20Adamovi=C4=87?=
 <slobodanadamovic@users.noreply.github.com>
Date: Fri, 31 May 2024 01:27:13 +0200
Subject: [PATCH 070/208] Expose API Key cache metrics (#109078)

This PR exposes API key cache stats (hit, miss, eviction and count) as
APM metrics for:

- Authentication Cache
  - `es.security.api_key.auth_cache.hit.total`
  - `es.security.api_key.auth_cache.miss.total`
  - `es.security.api_key.auth_cache.eviction.total`
  - `es.security.api_key.auth_cache.count.current`

- Doc Cache
  - `es.security.api_key.doc_cache.hit.total`
  - `es.security.api_key.doc_cache.miss.total`
  - `es.security.api_key.doc_cache.eviction.total`
  - `es.security.api_key.doc_cache.count.current`

- Role Descripto Cache
  - `es.security.api_key.role_descriptor_cache.hit.total`
  - `es.security.api_key.role_descriptor_cache.miss.total`
  - `es.security.api_key.role_descriptor_cache.eviction.total`
  - `es.security.api_key.role_descriptor_cache.count.current`
---
 docs/changelog/109078.yaml                    |   5 +
 .../xpack/security/Security.java              |  34 +-
 .../xpack/security/authc/ApiKeyService.java   |  53 ++-
 .../security/metric/SecurityCacheMetrics.java |  80 +++++
 .../logfile/LoggingAuditTrailFilterTests.java |   4 +-
 .../audit/logfile/LoggingAuditTrailTests.java |   4 +-
 .../security/authc/ApiKeyServiceTests.java    | 331 +++++++++++++++++-
 .../authc/AuthenticationServiceTests.java     |   3 +-
 .../support/SecondaryAuthenticatorTests.java  |   3 +-
 .../authz/store/CompositeRolesStoreTests.java |  16 +-
 10 files changed, 506 insertions(+), 27 deletions(-)
 create mode 100644 docs/changelog/109078.yaml
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/metric/SecurityCacheMetrics.java

diff --git a/docs/changelog/109078.yaml b/docs/changelog/109078.yaml
new file mode 100644
index 0000000000000..f602ee9b131bc
--- /dev/null
+++ b/docs/changelog/109078.yaml
@@ -0,0 +1,5 @@
+pr: 109078
+summary: Expose API Key cache metrics
+area: Authentication
+type: enhancement
+issues: []
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 84fa92bb7d2d4..283e8a03c1a95 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -56,6 +56,7 @@
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeMetadata;
@@ -408,6 +409,7 @@
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4ServerTransport;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.security.Provider;
@@ -619,6 +621,8 @@ public class Security extends Plugin
     // restart or master node change.
     private final AtomicInteger nodeLocalMigrationRetryCount = new AtomicInteger(0);
 
+    private final SetOnce<List<Closeable>> closableComponents = new SetOnce<>();
+
     public Security(Settings settings) {
         this(settings, Collections.emptyList());
     }
@@ -937,7 +941,8 @@ Collection<Object> createComponents(
             systemIndices.getMainIndexManager(),
             clusterService,
             cacheInvalidatorRegistry,
-            threadPool
+            threadPool,
+            telemetryProvider.getMeterRegistry()
         );
         components.add(apiKeyService);
 
@@ -1152,13 +1157,19 @@ Collection<Object> createComponents(
 
         cacheInvalidatorRegistry.validate();
 
-        this.reloadableComponents.set(
-            components.stream()
-                .filter(ReloadableSecurityComponent.class::isInstance)
-                .map(ReloadableSecurityComponent.class::cast)
-                .collect(Collectors.toUnmodifiableList())
-        );
+        final List<ReloadableSecurityComponent> reloadableComponents = new ArrayList<>();
+        final List<Closeable> closableComponents = new ArrayList<>();
+        for (Object component : components) {
+            if (component instanceof ReloadableSecurityComponent reloadable) {
+                reloadableComponents.add(reloadable);
+            }
+            if (component instanceof Closeable closeable) {
+                closableComponents.add(closeable);
+            }
+        }
 
+        this.reloadableComponents.set(List.copyOf(reloadableComponents));
+        this.closableComponents.set(List.copyOf(closableComponents));
         return components;
     }
 
@@ -2287,4 +2298,13 @@ List<ReservedClusterStateHandler<?>> reservedClusterStateHandlers() {
     OperatorPrivileges.OperatorPrivilegesService getOperatorPrivilegesService() {
         return operatorPrivilegesService.get();
     }
+
+    @Override
+    public void close() throws IOException {
+        if (enabled) {
+            if (closableComponents.get() != null) {
+                IOUtils.close(closableComponents.get());
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index 0b17bdce98858..233f5791a6452 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -67,6 +67,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.InstantiatingObjectParser;
@@ -103,6 +104,7 @@
 import org.elasticsearch.xpack.core.security.authz.store.RoleReference;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.metric.SecurityCacheMetrics;
 import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry;
 import org.elasticsearch.xpack.security.support.FeatureNotEnabledException;
 import org.elasticsearch.xpack.security.support.FeatureNotEnabledException.Feature;
@@ -149,7 +151,7 @@
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.Availability.SEARCH_SHARDS;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS;
 
-public class ApiKeyService {
+public class ApiKeyService implements Closeable {
 
     private static final Logger logger = LogManager.getLogger(ApiKeyService.class);
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ApiKeyService.class);
@@ -223,6 +225,8 @@ public class ApiKeyService {
     private final AtomicLong lastEvictionCheckedAt = new AtomicLong(0);
     private final LongAdder evictionCounter = new LongAdder();
 
+    private final List<AutoCloseable> cacheMetrics;
+
     @SuppressWarnings("this-escape")
     public ApiKeyService(
         Settings settings,
@@ -231,7 +235,8 @@ public ApiKeyService(
         SecurityIndexManager securityIndex,
         ClusterService clusterService,
         CacheInvalidatorRegistry cacheInvalidatorRegistry,
-        ThreadPool threadPool
+        ThreadPool threadPool,
+        MeterRegistry meterRegistry
     ) {
         this.clock = clock;
         this.client = client;
@@ -289,6 +294,39 @@ public void invalidateAll() {
             this.apiKeyAuthCache = null;
             this.apiKeyDocCache = null;
         }
+
+        if (enabled) {
+            final List<AutoCloseable> cacheMetrics = new ArrayList<>();
+            if (this.apiKeyAuthCache != null) {
+                cacheMetrics.addAll(
+                    SecurityCacheMetrics.registerAsyncCacheMetrics(
+                        meterRegistry,
+                        this.apiKeyAuthCache,
+                        SecurityCacheMetrics.CacheType.API_KEY_AUTH_CACHE
+                    )
+                );
+            }
+            if (this.apiKeyDocCache != null) {
+                cacheMetrics.addAll(
+                    SecurityCacheMetrics.registerAsyncCacheMetrics(
+                        meterRegistry,
+                        this.apiKeyDocCache.docCache,
+                        SecurityCacheMetrics.CacheType.API_KEY_DOCS_CACHE
+                    )
+                );
+                cacheMetrics.addAll(
+                    SecurityCacheMetrics.registerAsyncCacheMetrics(
+                        meterRegistry,
+                        this.apiKeyDocCache.roleDescriptorsBytesCache,
+                        SecurityCacheMetrics.CacheType.API_KEY_ROLE_DESCRIPTORS_CACHE
+                    )
+                );
+            }
+            this.cacheMetrics = List.copyOf(cacheMetrics);
+        } else {
+            this.cacheMetrics = List.of();
+        }
+
     }
 
     /**
@@ -1448,6 +1486,17 @@ public void crossClusterApiKeyUsageStats(ActionListener<Map<String, Object>> lis
         }
     }
 
+    @Override
+    public void close() {
+        cacheMetrics.forEach(metric -> {
+            try {
+                metric.close();
+            } catch (Exception e) {
+                logger.warn("metrics close() method should not throw Exception", e);
+            }
+        });
+    }
+
     // public class for testing
     public static final class ApiKeyCredentials implements AuthenticationToken, Closeable {
         private final String id;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/metric/SecurityCacheMetrics.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/metric/SecurityCacheMetrics.java
new file mode 100644
index 0000000000000..4dcffea8d43ef
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/metric/SecurityCacheMetrics.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.metric;
+
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.telemetry.metric.LongWithAttributes;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public final class SecurityCacheMetrics {
+
+    public static <K, V> List<AutoCloseable> registerAsyncCacheMetrics(MeterRegistry registry, Cache<K, V> cache, CacheType type) {
+        final List<AutoCloseable> metrics = new ArrayList<>();
+        metrics.add(
+            registry.registerLongAsyncCounter(
+                type.metricsPrefix + ".hit.total",
+                "Total number of cache hits.",
+                "count",
+                () -> new LongWithAttributes(cache.stats().getHits())
+            )
+        );
+        metrics.add(
+            registry.registerLongAsyncCounter(
+                type.metricsPrefix + ".miss.total",
+                "Total number of cache misses.",
+                "count",
+                () -> new LongWithAttributes(cache.stats().getMisses())
+            )
+        );
+        metrics.add(
+            registry.registerLongAsyncCounter(
+                type.metricsPrefix + ".eviction.total",
+                "Total number of cache evictions.",
+                "count",
+                () -> new LongWithAttributes(cache.stats().getEvictions())
+            )
+        );
+        metrics.add(
+            registry.registerLongGauge(
+                type.metricsPrefix + ".count.current",
+                "The current number of cache entries.",
+                "count",
+                () -> new LongWithAttributes(cache.count())
+            )
+        );
+        return metrics;
+    }
+
+    public enum CacheType {
+
+        API_KEY_AUTH_CACHE("es.security.api_key.auth_cache"),
+
+        API_KEY_DOCS_CACHE("es.security.api_key.doc_cache"),
+
+        API_KEY_ROLE_DESCRIPTORS_CACHE("es.security.api_key.role_descriptor_cache"),
+
+        ;
+
+        private final String metricsPrefix;
+
+        CacheType(String metricsPrefix) {
+            this.metricsPrefix = metricsPrefix;
+        }
+
+        public String metricsPrefix() {
+            return metricsPrefix;
+        }
+    }
+
+    private SecurityCacheMetrics() {
+        throw new IllegalAccessError();
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java
index 4f4e35e1a30c7..1a42914f98c10 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.HttpRequest;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.test.rest.FakeRestRequest.Builder;
@@ -124,7 +125,8 @@ public void init() throws Exception {
             mock(SecurityIndexManager.class),
             clusterService,
             mock(CacheInvalidatorRegistry.class),
-            mock(ThreadPool.class)
+            mock(ThreadPool.class),
+            MeterRegistry.NOOP
         );
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
index 22cd2816ec334..a3292a6ab5f4e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.test.rest.FakeRestRequest.Builder;
@@ -360,7 +361,8 @@ public void init() throws Exception {
             securityIndexManager,
             clusterService,
             mock(CacheInvalidatorRegistry.class),
-            mock(ThreadPool.class)
+            mock(ThreadPool.class),
+            MeterRegistry.NOOP
         );
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
index 44d0543cc7831..1dce6a038638b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
@@ -63,6 +63,9 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.telemetry.Measurement;
+import org.elasticsearch.telemetry.TestTelemetryPlugin;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLog;
@@ -115,6 +118,7 @@
 import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyDoc;
 import org.elasticsearch.xpack.security.authc.ApiKeyService.CachedApiKeyHashResult;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
+import org.elasticsearch.xpack.security.metric.SecurityCacheMetrics.CacheType;
 import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry;
 import org.elasticsearch.xpack.security.support.FeatureNotEnabledException;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
@@ -147,10 +151,12 @@
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import java.util.stream.LongStream;
 
 import static org.elasticsearch.TransportVersions.ROLE_REMOTE_CLUSTER_PRIVS;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
@@ -1120,7 +1126,21 @@ private Map<String, Object> mockKeyDocument(
         @Nullable List<RoleDescriptor> keyRoles,
         ApiKey.Type type
     ) throws IOException {
-        var apiKeyDoc = newApiKeyDocument(key, user, authUser, invalidated, expiry, keyRoles, type);
+        return mockKeyDocument(id, key, user, authUser, invalidated, expiry, keyRoles, type, List.of(SUPERUSER_ROLE_DESCRIPTOR));
+    }
+
+    private Map<String, Object> mockKeyDocument(
+        String id,
+        String key,
+        User user,
+        @Nullable User authUser,
+        boolean invalidated,
+        Duration expiry,
+        @Nullable List<RoleDescriptor> keyRoles,
+        ApiKey.Type type,
+        @Nullable List<RoleDescriptor> userRoles
+    ) throws IOException {
+        var apiKeyDoc = newApiKeyDocument(key, user, authUser, invalidated, expiry, keyRoles, type, userRoles);
         SecurityMocks.mockGetRequest(
             client,
             id,
@@ -1136,7 +1156,8 @@ private static Tuple<Map<String, Object>, Map<String, Object>> newApiKeyDocument
         boolean invalidated,
         Duration expiry,
         @Nullable List<RoleDescriptor> keyRoles,
-        ApiKey.Type type
+        ApiKey.Type type,
+        @Nullable List<RoleDescriptor> userRoles
     ) throws IOException {
         final Authentication authentication;
         if (authUser != null) {
@@ -1158,9 +1179,7 @@ private static Tuple<Map<String, Object>, Map<String, Object>> newApiKeyDocument
             getFastStoredHashAlgoForTests().hash(new SecureString(key.toCharArray())),
             "test",
             authentication,
-            type == ApiKey.Type.CROSS_CLUSTER
-                ? Set.of()
-                : ApiKeyService.removeUserRoleDescriptorDescriptions(Set.of(SUPERUSER_ROLE_DESCRIPTOR)),
+            type == ApiKey.Type.CROSS_CLUSTER ? Set.of() : ApiKeyService.removeUserRoleDescriptorDescriptions(Set.copyOf(userRoles)),
             Instant.now(),
             Instant.now().plus(expiry),
             keyRoles,
@@ -1485,6 +1504,292 @@ public void testApiKeyCache() throws IOException {
         assertThat(service.getFromCache(creds.getId()).success, is(true));
     }
 
+    public void testApiKeyAuthCacheHitAndMissMetrics() {
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
+        final MeterRegistry meterRegistry = telemetryPlugin.getTelemetryProvider(Settings.EMPTY).getMeterRegistry();
+
+        final long cacheSize = randomLongBetween(2, 8);
+        ApiKeyService service = createApiKeyService(
+            Settings.builder().put("xpack.security.authc.api_key.cache.max_keys", cacheSize).build(),
+            meterRegistry
+        );
+        final Cache<String, ListenableFuture<CachedApiKeyHashResult>> apiKeyAuthCache = service.getApiKeyAuthCache();
+
+        // sanity check - cache metrics should be all zeros
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_AUTH_CACHE,
+            Map.of("count", 0L, "hit", 0L, "miss", 0L, "eviction", 0L)
+        );
+
+        // fill the cache with random data
+        final String idPrefix = randomAlphaOfLength(20);
+        LongStream.range(0, cacheSize).forEach(i -> {
+            apiKeyAuthCache.put(idPrefix + i, new ListenableFuture<>());
+            collectAndAssertCacheMetrics(
+                telemetryPlugin,
+                CacheType.API_KEY_AUTH_CACHE,
+                Map.of("count", i + 1, "hit", 0L, "miss", 0L, "eviction", 0L)
+            );
+        });
+
+        // test hit metric collection
+        long numberOfHits = randomLongBetween(0, 5);
+        for (long i = 0L; i < numberOfHits; i++) {
+            var cacheEntry = apiKeyAuthCache.get(idPrefix + randomLongBetween(0, cacheSize - 1));
+            assertThat(cacheEntry, is(notNullValue()));
+        }
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_AUTH_CACHE,
+            Map.of("count", cacheSize, "hit", numberOfHits, "miss", 0L, "eviction", 0L)
+        );
+
+        // test miss metric collection
+        long numberOfMisses = randomLongBetween(0, 5);
+        for (long i = 0L; i < numberOfMisses; i++) {
+            var cacheEntry = apiKeyAuthCache.get(idPrefix + (cacheSize + randomLongBetween(0, 3)));
+            assertThat(cacheEntry, is(nullValue()));
+        }
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_AUTH_CACHE,
+            Map.of("count", cacheSize, "hit", numberOfHits, "miss", numberOfMisses, "eviction", 0L)
+        );
+    }
+
+    public void testApiKeyAuthCacheEvictionMetrics() {
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
+        final MeterRegistry meterRegistry = telemetryPlugin.getTelemetryProvider(Settings.EMPTY).getMeterRegistry();
+
+        final long cacheSize = randomLongBetween(2, 8);
+        ApiKeyService service = createApiKeyService(
+            Settings.builder().put("xpack.security.authc.api_key.cache.max_keys", cacheSize).build(),
+            meterRegistry
+        );
+        final Cache<String, ListenableFuture<CachedApiKeyHashResult>> apiKeyAuthCache = service.getApiKeyAuthCache();
+
+        // Fill the cache
+        final String idPrefix = randomAlphaOfLength(20);
+        final AtomicLong count = new AtomicLong(0);
+        LongStream.range(0, cacheSize).forEach(i -> apiKeyAuthCache.put(idPrefix + count.incrementAndGet(), new ListenableFuture<>()));
+
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_AUTH_CACHE,
+            Map.of("count", cacheSize, "hit", 0L, "miss", 0L, "eviction", 0L)
+        );
+
+        // putting a new entry in a full cache should evict one entry
+        apiKeyAuthCache.put(idPrefix + count.incrementAndGet(), new ListenableFuture<>());
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_AUTH_CACHE,
+            Map.of("count", cacheSize, "hit", 0L, "miss", 0L, "eviction", 1L)
+        );
+
+        // evict one more by adding a new entry to a full cache
+        apiKeyAuthCache.put(idPrefix + count.incrementAndGet(), new ListenableFuture<>());
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_AUTH_CACHE,
+            Map.of("count", cacheSize, "hit", 0L, "miss", 0L, "eviction", 2L)
+        );
+
+        // replacing existing entry should not change any metrics
+        apiKeyAuthCache.put(idPrefix + count.get(), new ListenableFuture<>());
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_AUTH_CACHE,
+            Map.of("count", cacheSize, "hit", 0L, "miss", 0L, "eviction", 2L)
+        );
+
+        // explicitly invalidated entry do not count as eviction - all cache metrics should stay unchanged
+        ListenableFuture<ApiKeyService.CachedApiKeyHashResult> future = new ListenableFuture<>();
+        apiKeyAuthCache.invalidate(idPrefix + count.get(), future);
+        future.onResponse(null);
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_AUTH_CACHE,
+            Map.of("count", cacheSize, "hit", 0L, "miss", 0L, "eviction", 2L)
+        );
+
+        // invalidating all entries does not count as eviction - eviction metrics should stay unchanged
+        apiKeyAuthCache.invalidateAll();
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_AUTH_CACHE,
+            Map.of("count", 0L, "hit", 0L, "miss", 0L, "eviction", 2L)
+        );
+    }
+
+    public void testApiKeyDocAndRoleDescriptorsCacheMetrics() throws Exception {
+        final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
+        final MeterRegistry meterRegistry = telemetryPlugin.getTelemetryProvider(Settings.EMPTY).getMeterRegistry();
+
+        // setting cache size to 1 in order to test evictions as well
+        // doc cache size = 1
+        // role descriptors size = 2
+        ApiKeyService service = createApiKeyService(
+            Settings.builder().put("xpack.security.authc.api_key.cache.max_keys", 1L).build(),
+            meterRegistry
+        );
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        final ApiKey.Type type = ApiKey.Type.REST;
+
+        // new API key document will be cached after its authentication
+        final String docId = randomAlphaOfLength(16);
+        final String apiKey = randomAlphaOfLength(16);
+
+        // both API key doc and role descriptor caches should be empty
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_DOCS_CACHE,
+            Map.of("count", 0L, "hit", 0L, "miss", 0L, "eviction", 0L)
+        );
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_ROLE_DESCRIPTORS_CACHE,
+            Map.of("count", 0L, "hit", 0L, "miss", 0L, "eviction", 0L)
+        );
+
+        final Map<String, Object> metadata = mockKeyDocument(
+            docId,
+            apiKey,
+            new User("hulk", "superuser"),
+            null,
+            false,
+            Duration.ofSeconds(3600),
+            List.of(randomRoleDescriptorWithWorkflowsRestriction()),
+            type
+        );
+        ApiKeyCredentials apiKeyCredentials = getApiKeyCredentials(docId, apiKey, type);
+        service.loadApiKeyAndValidateCredentials(threadContext, apiKeyCredentials, new PlainActionFuture<>());
+
+        // initial authentication fails to find API key doc and role descriptors in cache:
+        // - miss metrics should be increased because we first check the caches
+        // - counts should be increased because we cache entries after loading them
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_DOCS_CACHE,
+            Map.of("count", 1L, "hit", 0L, "miss", 1L, "eviction", 0L)
+        );
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_ROLE_DESCRIPTORS_CACHE,
+            Map.of("count", 2L, "hit", 0L, "miss", 2L, "eviction", 0L)
+        );
+
+        // fetching existing API key doc from cache to verify hit metrics
+        var cachedApiKeyDoc = service.getDocCache().get(docId);
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_DOCS_CACHE,
+            Map.of("count", 1L, "hit", 1L, "miss", 1L, "eviction", 0L)
+        );
+
+        // fetching existing role descriptors to verify hit metrics get collected
+        var roleDescriptorsBytes = service.getRoleDescriptorsBytesCache().get(cachedApiKeyDoc.roleDescriptorsHash);
+        assertNotNull(roleDescriptorsBytes);
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_ROLE_DESCRIPTORS_CACHE,
+            Map.of("count", 2L, "hit", 1L, "miss", 2L, "eviction", 0L)
+        );
+        var limitedByRoleDescriptorsBytes = service.getRoleDescriptorsBytesCache().get(cachedApiKeyDoc.limitedByRoleDescriptorsHash);
+        assertNotNull(limitedByRoleDescriptorsBytes);
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_ROLE_DESCRIPTORS_CACHE,
+            Map.of("count", 2L, "hit", 2L, "miss", 2L, "eviction", 0L)
+        );
+
+        // a different API Key to test evictions
+        final String docId2 = randomValueOtherThan(docId, () -> randomAlphaOfLength(16));
+        final String apiKey2 = randomValueOtherThan(apiKey, () -> randomAlphaOfLength(16));
+        ApiKeyCredentials apiKeyCredentials2 = getApiKeyCredentials(docId2, apiKey2, type);
+        final Map<String, Object> metadata2 = mockKeyDocument(
+            docId2,
+            apiKey2,
+            new User("spider-man", "monitoring_user"),
+            null,
+            false,
+            Duration.ofSeconds(3600),
+            List.of(randomRoleDescriptorWithWorkflowsRestriction(), randomRoleDescriptorWithRemotePrivileges()),
+            type,
+            List.of(randomRoleDescriptorWithRemotePrivileges())
+        );
+        service.loadApiKeyAndValidateCredentials(threadContext, apiKeyCredentials2, new PlainActionFuture<>());
+
+        // authenticating with second key will
+        // - fail to find API key doc and role descriptors in cache
+        // - cache both new doc and new roles
+        // - this will evict entries from both doc and roles cache
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_DOCS_CACHE,
+            Map.of("count", 1L, "hit", 1L, "miss", 2L, "eviction", 1L)
+        );
+        collectAndAssertCacheMetrics(
+            telemetryPlugin,
+            CacheType.API_KEY_ROLE_DESCRIPTORS_CACHE,
+            Map.of("count", 2L, "hit", 2L, "miss", 4L, "eviction", 2L)
+        );
+
+    }
+
+    private void assertCacheCount(TestTelemetryPlugin telemetryPlugin, CacheType type, long expectedCount) {
+        List<Measurement> metrics = telemetryPlugin.getLongGaugeMeasurement(type.metricsPrefix() + ".count.current");
+        final Long actual;
+        if (metrics.isEmpty()) {
+            actual = 0L;
+        } else {
+            actual = metrics.get(metrics.size() - 1).getLong();
+        }
+        assertThat(actual, equalTo(expectedCount));
+    }
+
+    private void assertCacheHits(TestTelemetryPlugin telemetryPlugin, CacheType type, long expectedHits) {
+        List<Measurement> metrics = telemetryPlugin.getLongAsyncCounterMeasurement(type.metricsPrefix() + ".hit.total");
+        final Long actual;
+        if (metrics.isEmpty()) {
+            actual = 0L;
+        } else {
+            actual = metrics.get(metrics.size() - 1).getLong();
+        }
+        assertThat(actual, equalTo(expectedHits));
+    }
+
+    private void assertCacheMisses(TestTelemetryPlugin telemetryPlugin, CacheType type, long expectedMisses) {
+        List<Measurement> metrics = telemetryPlugin.getLongAsyncCounterMeasurement(type.metricsPrefix() + ".miss.total");
+        final Long actual;
+        if (metrics.isEmpty()) {
+            actual = 0L;
+        } else {
+            actual = metrics.get(metrics.size() - 1).getLong();
+        }
+        assertThat(actual, equalTo(expectedMisses));
+    }
+
+    private void assertCacheEvictions(TestTelemetryPlugin telemetryPlugin, CacheType type, long expectedEvictions) {
+        List<Measurement> metrics = telemetryPlugin.getLongAsyncCounterMeasurement(type.metricsPrefix() + ".eviction.total");
+        final Long actual;
+        if (metrics.isEmpty()) {
+            actual = 0L;
+        } else {
+            actual = metrics.get(metrics.size() - 1).getLong();
+        }
+        assertThat(actual, equalTo(expectedEvictions));
+    }
+
+    private void collectAndAssertCacheMetrics(TestTelemetryPlugin telemetryPlugin, CacheType type, Map<String, Long> expected) {
+        telemetryPlugin.collect();
+        assertCacheCount(telemetryPlugin, type, expected.get("count"));
+        assertCacheHits(telemetryPlugin, type, expected.get("hit"));
+        assertCacheMisses(telemetryPlugin, type, expected.get("miss"));
+        assertCacheEvictions(telemetryPlugin, type, expected.get("eviction"));
+    }
+
     public void testApiKeyAuthCacheWillTraceLogOnEvictionDueToCacheSize() throws IllegalAccessException {
         final int cacheSize = randomIntBetween(2, 8);
         ApiKeyService service = createApiKeyService(
@@ -2716,7 +3021,8 @@ public void testCreateCrossClusterApiKeyMinVersionConstraint() {
             securityIndex,
             clusterService,
             cacheInvalidatorRegistry,
-            threadPool
+            threadPool,
+            MeterRegistry.NOOP
         );
 
         final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
@@ -2853,7 +3159,8 @@ public void testCreateOrUpdateApiKeyWithWorkflowsRestrictionForUnsupportedVersio
             securityIndex,
             clusterService,
             cacheInvalidatorRegistry,
-            threadPool
+            threadPool,
+            MeterRegistry.NOOP
         );
 
         final List<RoleDescriptor> roleDescriptorsWithWorkflowsRestriction = randomList(
@@ -2918,7 +3225,8 @@ public void testValidateOwnerUserRoleDescriptorsWithWorkflowsRestriction() {
             securityIndex,
             clusterService,
             cacheInvalidatorRegistry,
-            threadPool
+            threadPool,
+            MeterRegistry.NOOP
         );
 
         final Set<RoleDescriptor> userRoleDescriptorsWithWorkflowsRestriction = randomSet(
@@ -3075,6 +3383,10 @@ private ApiKeyService createApiKeyService() {
     }
 
     private ApiKeyService createApiKeyService(Settings baseSettings) {
+        return createApiKeyService(baseSettings, MeterRegistry.NOOP);
+    }
+
+    private ApiKeyService createApiKeyService(Settings baseSettings, MeterRegistry meterRegistry) {
         final Settings settings = Settings.builder()
             .put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true)
             .put(baseSettings)
@@ -3093,7 +3405,8 @@ private ApiKeyService createApiKeyService(Settings baseSettings) {
             securityIndex,
             ClusterServiceUtils.createClusterService(threadPool, clusterSettings),
             cacheInvalidatorRegistry,
-            threadPool
+            threadPool,
+            meterRegistry
         );
         if ("0s".equals(settings.get(ApiKeyService.CACHE_TTL_SETTING.getKey()))) {
             verify(cacheInvalidatorRegistry, never()).registerCacheInvalidator(eq("api_key"), any());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index 963ff62bc137d..330eecc1563e2 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -332,7 +332,8 @@ public void init() throws Exception {
             securityIndex,
             clusterService,
             mock(CacheInvalidatorRegistry.class),
-            threadPool
+            threadPool,
+            MeterRegistry.NOOP
         );
         tokenService = new TokenService(
             settings,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java
index 9a8bb5764ce2d..0b29b46b19b36 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java
@@ -138,7 +138,8 @@ public void setupMocks() throws Exception {
             securityIndex,
             clusterService,
             mock(CacheInvalidatorRegistry.class),
-            threadPool
+            threadPool,
+            MeterRegistry.NOOP
         );
         final ServiceAccountService serviceAccountService = mock(ServiceAccountService.class);
         doAnswer(invocationOnMock -> {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
index 9d73f7e688f7d..c0b24a26010a7 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.license.LicenseStateListener;
 import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLog;
 import org.elasticsearch.test.TransportVersionUtils;
@@ -1948,7 +1949,8 @@ public void testApiKeyAuthUsesApiKeyService() throws Exception {
                 mock(SecurityIndexManager.class),
                 clusterService,
                 mock(CacheInvalidatorRegistry.class),
-                mock(ThreadPool.class)
+                mock(ThreadPool.class),
+                MeterRegistry.NOOP
             )
         );
         NativePrivilegeStore nativePrivStore = mock(NativePrivilegeStore.class);
@@ -2031,7 +2033,8 @@ public void testApiKeyAuthUsesApiKeyServiceWithScopedRole() throws Exception {
                 mock(SecurityIndexManager.class),
                 clusterService,
                 mock(CacheInvalidatorRegistry.class),
-                mock(ThreadPool.class)
+                mock(ThreadPool.class),
+                MeterRegistry.NOOP
             )
         );
         NativePrivilegeStore nativePrivStore = mock(NativePrivilegeStore.class);
@@ -2131,7 +2134,8 @@ public void testGetRoleForCrossClusterAccessAuthentication() throws Exception {
                 mock(SecurityIndexManager.class),
                 clusterService,
                 mock(CacheInvalidatorRegistry.class),
-                mock(ThreadPool.class)
+                mock(ThreadPool.class),
+                MeterRegistry.NOOP
             )
         );
         final NativePrivilegeStore nativePrivStore = mock(NativePrivilegeStore.class);
@@ -2289,7 +2293,8 @@ public void testGetRoleForWorkflowWithRestriction() {
             mock(SecurityIndexManager.class),
             clusterService,
             mock(CacheInvalidatorRegistry.class),
-            mock(ThreadPool.class)
+            mock(ThreadPool.class),
+            MeterRegistry.NOOP
         );
         final NativePrivilegeStore privilegeStore = mock(NativePrivilegeStore.class);
         doAnswer((invocationOnMock) -> {
@@ -2401,7 +2406,8 @@ public void testGetRoleForWorkflowWithoutRestriction() {
             mock(SecurityIndexManager.class),
             clusterService,
             mock(CacheInvalidatorRegistry.class),
-            mock(ThreadPool.class)
+            mock(ThreadPool.class),
+            MeterRegistry.NOOP
         );
         final NativePrivilegeStore privilegeStore = mock(NativePrivilegeStore.class);
         doAnswer((invocationOnMock) -> {

From 3a5e94011e5c624df63535757adbb343187441dc Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 31 May 2024 10:44:43 +0200
Subject: [PATCH 071/208] Do not request gaps to fill when the sub range is
 already completed (#109212)

When the range to read (subRange) is already completed, we can
optimize by returning early without asking for the range to write
(range) to be fully filled and save one or more fetch requests.
---
 .../blobcache/common/SparseFileTracker.java   |  9 +--
 .../common/SparseFileTrackerTests.java        | 61 ++++++++++++++++++-
 2 files changed, 65 insertions(+), 5 deletions(-)

diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/SparseFileTracker.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/SparseFileTracker.java
index 17e48c91a4eab..6e6a11fbddc93 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/SparseFileTracker.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/SparseFileTracker.java
@@ -139,9 +139,10 @@ private long computeLengthOfRanges() {
     }
 
     /**
-     * Called before reading a range from the file to ensure that this range is present. Returns a list of gaps for the caller to fill. The
-     * range from the file is defined by {@code range} but the listener is executed as soon as a (potentially smaller) sub range
-     * {@code subRange} becomes available.
+     * Called before reading a range from the file to ensure that this range is present. Returns a list of gaps for the caller to fill,
+     * unless the {@code subRange} is already present in which case the listener is executed immediately without returning gaps. The range
+     * from the file is defined by {@code range} but the listener is executed as soon as a (potentially smaller) sub range {@code subRange}
+     * becomes available.
      *
      * @param range    A ByteRange that contains the (inclusive) start and (exclusive) end of the desired range
      * @param subRange A ByteRange that contains the (inclusive) start and (exclusive) end of the listener's range
@@ -173,7 +174,7 @@ public List<Gap> waitForRange(final ByteRange range, final ByteRange subRange, f
             );
         }
 
-        if (complete >= range.end()) {
+        if (subRange.end() <= complete) {
             listener.onResponse(null);
             return List.of();
         }
diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
index 6f1635e67ca4b..8808e7943f706 100644
--- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
@@ -10,6 +10,8 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionTestUtils;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.blobcache.BlobCacheUtils;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.test.ESTestCase;
 
@@ -22,7 +24,9 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiFunction;
 import java.util.function.Consumer;
+import java.util.stream.LongStream;
 
 import static org.elasticsearch.blobcache.BlobCacheTestUtils.mergeContiguousRanges;
 import static org.elasticsearch.blobcache.BlobCacheTestUtils.randomRanges;
@@ -35,6 +39,7 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class SparseFileTrackerTests extends ESTestCase {
@@ -119,7 +124,50 @@ public void testInvalidRange() {
         }
     }
 
-    public void testCallsListenerWhenRangeIsCompleted() {
+    public void testListenerCompletedImmediatelyWhenSubRangeIsAvailable() {
+        final byte[] bytes = new byte[randomIntBetween(8, 1024)];
+        final var tracker = new SparseFileTracker(getTestName(), bytes.length);
+
+        // wraps a future to assert that the sub range bytes are available
+        BiFunction<ByteRange, PlainActionFuture<Void>, ActionListener<Void>> wrapper = (range, future) -> ActionListener.runBefore(
+            future,
+            () -> LongStream.range(range.start(), range.end())
+                .forEach(pos -> assertThat(bytes[BlobCacheUtils.toIntBytes(pos)], equalTo(AVAILABLE)))
+        );
+
+        var completeUpTo = randomIntBetween(2, bytes.length);
+        {
+            long subRangeStart = randomLongBetween(0, completeUpTo - 2);
+            long subRangeEnd = randomLongBetween(subRangeStart + 1, completeUpTo - 1);
+            var subRange = ByteRange.of(subRangeStart, subRangeEnd);
+            var range = ByteRange.of(0, completeUpTo);
+            var future = new PlainActionFuture<Void>();
+
+            var gaps = tracker.waitForRange(range, subRange, wrapper.apply(subRange, future));
+            assertThat(future.isDone(), equalTo(false));
+            assertThat(gaps, notNullValue());
+            assertThat(gaps, hasSize(1));
+
+            fillGap(bytes, gaps.get(0));
+
+            assertThat(future.isDone(), equalTo(true));
+        }
+        {
+            long subRangeStart = randomLongBetween(0L, Math.max(0L, completeUpTo - 1));
+            long subRangeEnd = randomLongBetween(subRangeStart, completeUpTo);
+            var subRange = ByteRange.of(subRangeStart, subRangeEnd);
+
+            var range = ByteRange.of(randomLongBetween(0L, subRangeStart), randomLongBetween(subRangeEnd, bytes.length));
+            var future = new PlainActionFuture<Void>();
+
+            var gaps = tracker.waitForRange(range, subRange, wrapper.apply(subRange, future));
+            assertThat(future.isDone(), equalTo(true));
+            assertThat(gaps, notNullValue());
+            assertThat(gaps, hasSize(0));
+        }
+    }
+
+    public void testCallsListenerWhenWholeRangeIsAvailable() {
         final byte[] fileContents = new byte[between(0, 1000)];
         final SparseFileTracker sparseFileTracker = new SparseFileTracker("test", fileContents.length);
 
@@ -544,4 +592,15 @@ private static boolean processGap(byte[] fileContents, SparseFileTracker.Gap gap
             return true;
         }
     }
+
+    private static void fillGap(byte[] fileContents, SparseFileTracker.Gap gap) {
+        for (long i = gap.start(); i < gap.end(); i++) {
+            assertThat(fileContents[toIntBytes(i)], equalTo(UNAVAILABLE));
+        }
+        for (long i = gap.start(); i < gap.end(); i++) {
+            fileContents[toIntBytes(i)] = AVAILABLE;
+            gap.onProgress(i + 1L);
+        }
+        gap.onCompletion();
+    }
 }

From 29ccca3ddf1081565cc1c9ac549f95afedb5c00a Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 31 May 2024 10:52:31 +0200
Subject: [PATCH 072/208] Adjust index.mapping.dimension_fields.limit index
 setting default in docs. (#109091)

---
 docs/reference/data-streams/tsds-index-settings.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/data-streams/tsds-index-settings.asciidoc b/docs/reference/data-streams/tsds-index-settings.asciidoc
index 6dd902ff1c3b0..3ecfc60c90f58 100644
--- a/docs/reference/data-streams/tsds-index-settings.asciidoc
+++ b/docs/reference/data-streams/tsds-index-settings.asciidoc
@@ -60,5 +60,5 @@ information, refer to <<dimension-based-routing>>.
 `index.mapping.dimension_fields.limit`::
 (<<dynamic-index-settings,Dynamic>>, integer)
 Maximum number of <<time-series-dimension,time series dimensions>> for the
-index. Defaults to `21`.
+index. Defaults to `32768`.
 // end::dimensions-limit[]

From 98e02c3c878c11c2802da039bb74f47710ce6651 Mon Sep 17 00:00:00 2001
From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com>
Date: Fri, 31 May 2024 10:54:59 +0200
Subject: [PATCH 073/208] Fix warning message validation in failure store YAML
 test (#109197)

---
 .../test/data_stream/200_rollover_failure_store.yml           | 2 +-
 muted-tests.yml                                               | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml
index 11889a3153a98..8cdfe3d97bbb8 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml
@@ -218,7 +218,7 @@ teardown:
 
   - do:
       allowed_warnings:
-        - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
+        - "index template [my-template] has index patterns [data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
       indices.put_index_template:
         name: my-template
         body:
diff --git a/muted-tests.yml b/muted-tests.yml
index 303e0196782ad..40907992f58a3 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -35,10 +35,6 @@ tests:
 - class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108993"
   method: "testTrainedModelInference"
-- class: "org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109154"
-  method: "test {p0=data_stream/200_rollover_failure_store/Lazily roll over a data\
-    \ stream's failure store after an ingest failure}"
 - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109188"
   method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\

From 42f1e1d4d31ecb15d8bd167fc7381f71908bc6e1 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 31 May 2024 13:06:53 +0200
Subject: [PATCH 074/208] [Test] Fix
 SparseFileTrackerTests.testCallsListenerWhenRangeIsAvailable failing
 (#109239)

Test required some adjustments now listeners are
not completed on progress update if they are waiting
up to the end of the gap.

Relates #108095
Closes #109237
---
 .../common/SparseFileTrackerTests.java        | 43 ++++++++++++++-----
 1 file changed, 32 insertions(+), 11 deletions(-)

diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
index 8808e7943f706..5973b90c814d5 100644
--- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
@@ -326,9 +326,14 @@ public void testCallsListenerWhenRangeIsAvailable() {
                 assertThat(gap.start(), greaterThanOrEqualTo(range.start()));
                 assertThat(gap.end(), lessThanOrEqualTo(range.end()));
 
+                final boolean completeBeforeEndOfGap = triggeringProgress < gap.end() - 1L; // gap.end is exclusive
+                long from = gap.start();
+                long written = 0L;
+
                 for (long i = gap.start(); i < gap.end(); i++) {
                     assertThat(fileContents[toIntBytes(i)], equalTo(UNAVAILABLE));
                     fileContents[toIntBytes(i)] = AVAILABLE;
+                    written += 1L;
                     if (triggeringProgress == i) {
                         assertFalse(expectNotification.getAndSet(true));
                     }
@@ -342,19 +347,35 @@ public void testCallsListenerWhenRangeIsAvailable() {
                         equalTo(triggeringProgress < i)
                     );
 
-                    gap.onProgress(i + 1L);
+                    long progress = from + written;
+                    gap.onProgress(progress);
+
+                    if (completeBeforeEndOfGap) {
+                        assertThat(
+                            "Listener should not have been called before ["
+                                + triggeringProgress
+                                + "] is reached, but it was triggered after progress got updated to ["
+                                + i
+                                + ']',
+                            wasNotified.get() && waitIfPendingWasNotified.get(),
+                            equalTo(triggeringProgress < progress)
+                        );
+                    } else {
+                        assertThat(
+                            "Listener should not have been called before gap  ["
+                                + gap
+                                + "] is completed, but it was triggered after progress got updated to ["
+                                + i
+                                + ']',
+                            wasNotified.get() && waitIfPendingWasNotified.get(),
+                            equalTo(false)
+                        );
+                    }
 
-                    assertThat(
-                        "Listener should not have been called before ["
-                            + triggeringProgress
-                            + "] is reached, but it was triggered after progress got updated to ["
-                            + i
-                            + ']',
-                        wasNotified.get() && waitIfPendingWasNotified.get(),
-                        equalTo(triggeringProgress < i + 1L)
-                    );
+                    if (progress == gap.end()) {
+                        gap.onCompletion();
+                    }
                 }
-                gap.onCompletion();
 
                 assertThat(
                     "Listener should not have been called before ["

From 0177a0c8ae2e1cfccb3e2781679f3836708dbff9 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 31 May 2024 12:49:44 +0100
Subject: [PATCH 075/208] Fix trappy timeout in allocation explain API
 (#109240)

Relates #107984
---
 docs/changelog/109240.yaml                    |  5 ++
 .../api/cluster.allocation_explain.json       |  4 +
 .../ClusterAllocationExplainIT.java           | 37 +++------
 .../cluster/PrevalidateShardPathIT.java       | 16 ++--
 .../cluster/routing/AllocationIdIT.java       | 17 ++---
 .../RemoveCorruptedShardDataCommandIT.java    | 54 ++-----------
 .../index/store/CorruptedTranslogIT.java      | 12 +--
 .../ClusterAllocationExplainRequest.java      | 16 ++--
 ...lusterAllocationExplainRequestBuilder.java | 75 -------------------
 .../client/internal/ClusterAdminClient.java   | 16 ----
 .../RestClusterAllocationExplainAction.java   | 20 ++---
 .../ClusterAllocationExplainActionTests.java  | 30 +++++---
 .../ClusterAllocationExplainRequestTests.java |  2 +
 .../ClusterAllocationExplanationUtils.java    | 32 ++++++++
 .../elasticsearch/test/ESIntegTestCase.java   |  8 +-
 .../ccr/PrimaryFollowerAllocationIT.java      | 11 +--
 .../DataTierAllocationDeciderIT.java          |  3 +-
 ...ierShardAvailabilityHealthIndicatorIT.java |  9 +--
 .../xpack/ilm/DataTiersMigrationsTests.java   | 12 ++-
 .../FrozenSearchableSnapshotsIntegTests.java  | 12 +--
 .../SearchableSnapshotsIntegTests.java        | 10 +--
 ...tiallyCachedShardAllocationIntegTests.java | 25 +------
 .../xpack/shutdown/NodeShutdownShardsIT.java  | 20 +----
 23 files changed, 153 insertions(+), 293 deletions(-)
 create mode 100644 docs/changelog/109240.yaml
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationUtils.java

diff --git a/docs/changelog/109240.yaml b/docs/changelog/109240.yaml
new file mode 100644
index 0000000000000..a9fad3abdc47f
--- /dev/null
+++ b/docs/changelog/109240.yaml
@@ -0,0 +1,5 @@
+pr: 109240
+summary: Fix trappy timeout in allocation explain API
+area: Allocation
+type: bug
+issues: []
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
index e828af8a569ed..a3922033ec2a8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
@@ -22,6 +22,10 @@
       ]
     },
     "params":{
+      "master_timeout":{
+        "type":"time",
+        "description":"Timeout for connection to master node"
+      },
       "include_yes_decisions":{
         "type":"boolean",
         "description":"Return 'YES' decisions in explanation (default: false)"
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
index 1c358fe06b68f..26b33acfcbe98 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
@@ -158,12 +158,7 @@ public void testUnassignedReplicaDelayedAllocation() throws Exception {
         // wait till we have passed any pending shard data fetching
         assertEquals(
             AllocationDecision.ALLOCATION_DELAYED,
-            clusterAdmin().prepareAllocationExplain()
-                .setIndex("idx")
-                .setShard(0)
-                .setPrimary(false)
-                .get()
-                .getExplanation()
+            ClusterAllocationExplanationUtils.getClusterAllocationExplanation(client(), "idx", 0, false)
                 .getShardAllocationDecision()
                 .getAllocateDecision()
                 .getAllocationDecision()
@@ -1076,12 +1071,12 @@ public void testCannotAllocateStaleReplicaExplanation() throws Exception {
 
         // wait until the system has fetched shard data and we know there is no valid shard copy
         assertBusy(() -> {
-            ClusterAllocationExplanation explanation = clusterAdmin().prepareAllocationExplain()
-                .setIndex("idx")
-                .setShard(0)
-                .setPrimary(true)
-                .get()
-                .getExplanation();
+            ClusterAllocationExplanation explanation = ClusterAllocationExplanationUtils.getClusterAllocationExplanation(
+                client(),
+                "idx",
+                0,
+                true
+            );
             assertTrue(explanation.getShardAllocationDecision().getAllocateDecision().isDecisionTaken());
             assertEquals(
                 AllocationDecision.NO_VALID_SHARD_COPY,
@@ -1223,19 +1218,11 @@ private ClusterAllocationExplanation runExplain(boolean primary, boolean include
         return runExplain(primary, null, includeYesDecisions, includeDiskInfo);
     }
 
-    private ClusterAllocationExplanation runExplain(boolean primary, String nodeId, boolean includeYesDecisions, boolean includeDiskInfo)
-        throws Exception {
-
-        ClusterAllocationExplanation explanation = admin().cluster()
-            .prepareAllocationExplain()
-            .setIndex("idx")
-            .setShard(0)
-            .setPrimary(primary)
-            .setIncludeYesDecisions(includeYesDecisions)
-            .setIncludeDiskInfo(includeDiskInfo)
-            .setCurrentNode(nodeId)
-            .get()
-            .getExplanation();
+    private ClusterAllocationExplanation runExplain(boolean primary, String nodeId, boolean includeYesDecisions, boolean includeDiskInfo) {
+        final var request = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT, "idx", 0, primary, nodeId);
+        request.includeYesDecisions(includeYesDecisions);
+        request.includeDiskInfo(includeDiskInfo);
+        final var explanation = safeGet(client().execute(TransportClusterAllocationExplainAction.TYPE, request)).getExplanation();
         if (logger.isDebugEnabled()) {
             logger.debug("--> explain json output: \n{}", Strings.toString(explanation, true, true));
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java
index 77bcaf1e1970c..dd701244756cf 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java
@@ -9,8 +9,7 @@
 package org.elasticsearch.cluster;
 
 import org.apache.lucene.tests.util.LuceneTestCase;
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest;
-import org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction;
+import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils;
 import org.elasticsearch.action.admin.cluster.node.shutdown.NodePrevalidateShardPathResponse;
 import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateShardPathRequest;
 import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateShardPathResponse;
@@ -90,13 +89,12 @@ public void testCheckShards() throws Exception {
                         .filter(s -> node2ShardIds.contains(s.shardId()))
                         .filter(s -> s.currentNodeId().equals(node2Id))
                         .toList()) {
-                        var explanation = client().execute(
-                            TransportClusterAllocationExplainAction.TYPE,
-                            new ClusterAllocationExplainRequest().setIndex(node2Shard.getIndexName())
-                                .setCurrentNode(node2Shard.currentNodeId())
-                                .setShard(node2Shard.id())
-                                .setPrimary(node2Shard.primary())
-                        ).get();
+                        var explanation = ClusterAllocationExplanationUtils.getClusterAllocationExplanation(
+                            client(),
+                            node2Shard.getIndexName(),
+                            node2Shard.id(),
+                            node2Shard.primary()
+                        );
                         logger.info(
                             "Shard: {} is still located on relocation source node: {}. Allocation explanation: {}",
                             node2Shard.shardId(),
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java
index 012cb826a4403..784a6e8f419c8 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java
@@ -8,14 +8,13 @@
 
 package org.elasticsearch.cluster.routing;
 
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
+import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
-import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
@@ -199,14 +198,12 @@ private void putFakeCorruptionMarker(IndexSettings indexSettings, ShardId shardI
 
     private void checkNoValidShardCopy(String indexName, ShardId shardId) throws Exception {
         assertBusy(() -> {
-            final ClusterAllocationExplanation explanation = clusterAdmin().prepareAllocationExplain()
-                .setIndex(indexName)
-                .setShard(shardId.id())
-                .setPrimary(true)
-                .get()
-                .getExplanation();
-
-            final ShardAllocationDecision shardAllocationDecision = explanation.getShardAllocationDecision();
+            final var shardAllocationDecision = ClusterAllocationExplanationUtils.getClusterAllocationExplanation(
+                client(),
+                indexName,
+                shardId.id(),
+                true
+            ).getShardAllocationDecision();
             assertThat(shardAllocationDecision.isDecisionTaken(), equalTo(true));
             assertThat(
                 shardAllocationDecision.getAllocateDecision().getAllocationDecision(),
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java
index 6c691c0a14440..f43aaf0bacad4 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java
@@ -19,7 +19,6 @@
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.NativeFSLockFactory;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
@@ -37,7 +36,6 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
-import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -78,6 +76,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
+import static org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils.getClusterAllocationExplanation;
 import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -173,14 +172,7 @@ public Settings onNodeStopped(String nodeName) throws Exception {
 
         // shard should be failed due to a corrupted index
         assertBusy(() -> {
-            final ClusterAllocationExplanation explanation = clusterAdmin().prepareAllocationExplain()
-                .setIndex(indexName)
-                .setShard(0)
-                .setPrimary(true)
-                .get()
-                .getExplanation();
-
-            final ShardAllocationDecision shardAllocationDecision = explanation.getShardAllocationDecision();
+            final var shardAllocationDecision = getClusterAllocationExplanation(client(), indexName, 0, true).getShardAllocationDecision();
             assertThat(shardAllocationDecision.isDecisionTaken(), equalTo(true));
             assertThat(
                 shardAllocationDecision.getAllocateDecision().getAllocationDecision(),
@@ -219,14 +211,7 @@ public Settings onNodeStopped(String nodeName) throws Exception {
 
         // there is only _stale_ primary (due to new allocation id)
         assertBusy(() -> {
-            final ClusterAllocationExplanation explanation = clusterAdmin().prepareAllocationExplain()
-                .setIndex(indexName)
-                .setShard(0)
-                .setPrimary(true)
-                .get()
-                .getExplanation();
-
-            final ShardAllocationDecision shardAllocationDecision = explanation.getShardAllocationDecision();
+            final var shardAllocationDecision = getClusterAllocationExplanation(client(), indexName, 0, true).getShardAllocationDecision();
             assertThat(shardAllocationDecision.isDecisionTaken(), equalTo(true));
             assertThat(
                 shardAllocationDecision.getAllocateDecision().getAllocationDecision(),
@@ -237,13 +222,7 @@ public Settings onNodeStopped(String nodeName) throws Exception {
         clusterAdmin().prepareReroute().add(new AllocateStalePrimaryAllocationCommand(indexName, 0, nodeId, true)).get();
 
         assertBusy(() -> {
-            final ClusterAllocationExplanation explanation = clusterAdmin().prepareAllocationExplain()
-                .setIndex(indexName)
-                .setShard(0)
-                .setPrimary(true)
-                .get()
-                .getExplanation();
-
+            final var explanation = getClusterAllocationExplanation(client(), indexName, 0, true);
             assertThat(explanation.getCurrentNode(), notNullValue());
             assertThat(explanation.getShardState(), equalTo(ShardRoutingState.STARTED));
         });
@@ -331,13 +310,7 @@ public Settings onNodeStopped(String nodeName) throws Exception {
 
         // all shards should be failed due to a corrupted translog
         assertBusy(() -> {
-            final UnassignedInfo unassignedInfo = clusterAdmin().prepareAllocationExplain()
-                .setIndex(indexName)
-                .setShard(0)
-                .setPrimary(true)
-                .get()
-                .getExplanation()
-                .getUnassignedInfo();
+            final UnassignedInfo unassignedInfo = getClusterAllocationExplanation(client(), indexName, 0, true).getUnassignedInfo();
             assertThat(unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
             assertThat(ExceptionsHelper.unwrap(unassignedInfo.getFailure(), TranslogCorruptedException.class), not(nullValue()));
         });
@@ -392,14 +365,7 @@ public Settings onNodeStopped(String nodeName) throws Exception {
 
         // there is only _stale_ primary (due to new allocation id)
         assertBusy(() -> {
-            final ClusterAllocationExplanation explanation = clusterAdmin().prepareAllocationExplain()
-                .setIndex(indexName)
-                .setShard(0)
-                .setPrimary(true)
-                .get()
-                .getExplanation();
-
-            final ShardAllocationDecision shardAllocationDecision = explanation.getShardAllocationDecision();
+            final var shardAllocationDecision = getClusterAllocationExplanation(client(), indexName, 0, true).getShardAllocationDecision();
             assertThat(shardAllocationDecision.isDecisionTaken(), equalTo(true));
             assertThat(
                 shardAllocationDecision.getAllocateDecision().getAllocationDecision(),
@@ -410,13 +376,7 @@ public Settings onNodeStopped(String nodeName) throws Exception {
         clusterAdmin().prepareReroute().add(new AllocateStalePrimaryAllocationCommand(indexName, 0, primaryNodeId, true)).get();
 
         assertBusy(() -> {
-            final ClusterAllocationExplanation explanation = clusterAdmin().prepareAllocationExplain()
-                .setIndex(indexName)
-                .setShard(0)
-                .setPrimary(true)
-                .get()
-                .getExplanation();
-
+            final var explanation = getClusterAllocationExplanation(client(), indexName, 0, true);
             assertThat(explanation.getCurrentNode(), notNullValue());
             assertThat(explanation.getShardState(), equalTo(ShardRoutingState.STARTED));
         });
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
index 9618dcf761be9..ac5a10d246cfc 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.index.store;
 
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -32,6 +31,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 
+import static org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils.getClusterAllocationExplanation;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
@@ -79,13 +79,9 @@ public void onAllNodesStopped() throws Exception {
 
         assertBusy(() -> {
             // assertBusy since the shard starts out unassigned with reason CLUSTER_RECOVERED, then it's assigned, and then it fails.
-            final ClusterAllocationExplainResponse allocationExplainResponse = clusterAdmin().prepareAllocationExplain()
-                .setIndex("test")
-                .setShard(0)
-                .setPrimary(true)
-                .get();
-            final String description = Strings.toString(allocationExplainResponse.getExplanation());
-            final UnassignedInfo unassignedInfo = allocationExplainResponse.getExplanation().getUnassignedInfo();
+            final var allocationExplainResponse = getClusterAllocationExplanation(client(), "test", 0, true);
+            final var description = Strings.toString(allocationExplainResponse);
+            final var unassignedInfo = allocationExplainResponse.getUnassignedInfo();
             assertThat(description, unassignedInfo, not(nullValue()));
             assertThat(description, unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
             var failure = unassignedInfo.getFailure();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
index 91561814fea1a..11731c0ccade5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
@@ -48,8 +49,8 @@ public class ClusterAllocationExplainRequest extends MasterNodeRequest<ClusterAl
     /**
      * Create a new allocation explain request to explain any unassigned shard in the cluster.
      */
-    public ClusterAllocationExplainRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public ClusterAllocationExplainRequest(TimeValue masterNodeTimeout) {
+        super(masterNodeTimeout);
         this.index = null;
         this.shard = null;
         this.primary = null;
@@ -70,11 +71,10 @@ public ClusterAllocationExplainRequest(StreamInput in) throws IOException {
      * Create a new allocation explain request. If {@code primary} is false, the first unassigned replica
      * will be picked for explanation. If no replicas are unassigned, the first assigned replica will
      * be explained.
-     *
-     * Package private for testing.
      */
-    ClusterAllocationExplainRequest(String index, int shard, boolean primary, @Nullable String currentNode) {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    // Package private for testing.
+    ClusterAllocationExplainRequest(TimeValue masterNodeTimeout, String index, int shard, boolean primary, @Nullable String currentNode) {
+        super(masterNodeTimeout);
         this.index = index;
         this.shard = shard;
         this.primary = primary;
@@ -231,7 +231,7 @@ public String toString() {
         return sb.toString();
     }
 
-    public static ClusterAllocationExplainRequest parse(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new ClusterAllocationExplainRequest(), null);
+    public static ClusterAllocationExplainRequest parse(ClusterAllocationExplainRequest request, XContentParser parser) throws IOException {
+        return PARSER.parse(parser, request, null);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
deleted file mode 100644
index 3053ebe1f3db9..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action.admin.cluster.allocation;
-
-import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-
-/**
- * Builder for requests to explain the allocation of a shard in the cluster
- */
-public class ClusterAllocationExplainRequestBuilder extends MasterNodeOperationRequestBuilder<
-    ClusterAllocationExplainRequest,
-    ClusterAllocationExplainResponse,
-    ClusterAllocationExplainRequestBuilder> {
-
-    public ClusterAllocationExplainRequestBuilder(ElasticsearchClient client) {
-        super(client, TransportClusterAllocationExplainAction.TYPE, new ClusterAllocationExplainRequest());
-    }
-
-    /** The index name to use when finding the shard to explain */
-    public ClusterAllocationExplainRequestBuilder setIndex(String index) {
-        request.setIndex(index);
-        return this;
-    }
-
-    /** The shard number to use when finding the shard to explain */
-    public ClusterAllocationExplainRequestBuilder setShard(int shard) {
-        request.setShard(shard);
-        return this;
-    }
-
-    /** Whether the primary or replica should be explained */
-    public ClusterAllocationExplainRequestBuilder setPrimary(boolean primary) {
-        request.setPrimary(primary);
-        return this;
-    }
-
-    /** Whether to include "YES" decider decisions in the response instead of only "NO" decisions */
-    public ClusterAllocationExplainRequestBuilder setIncludeYesDecisions(boolean includeYesDecisions) {
-        request.includeYesDecisions(includeYesDecisions);
-        return this;
-    }
-
-    /** Whether to include information about the gathered disk information of nodes in the cluster */
-    public ClusterAllocationExplainRequestBuilder setIncludeDiskInfo(boolean includeDiskInfo) {
-        request.includeDiskInfo(includeDiskInfo);
-        return this;
-    }
-
-    /**
-     * Requests the explain API to explain an already assigned replica shard currently allocated to
-     * the given node.
-     */
-    public ClusterAllocationExplainRequestBuilder setCurrentNode(String currentNode) {
-        request.setCurrentNode(currentNode);
-        return this;
-    }
-
-    /**
-     * Signal that the first unassigned shard should be used
-     */
-    public ClusterAllocationExplainRequestBuilder useAnyUnassignedShard() {
-        request.setIndex(null);
-        request.setShard(null);
-        request.setPrimary(null);
-        return this;
-    }
-
-}
diff --git a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java
index daae078ed9a68..d2ac07cf16479 100644
--- a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java
+++ b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java
@@ -13,10 +13,6 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest;
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequestBuilder;
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
-import org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
@@ -455,18 +451,6 @@ public SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference sou
         return new SimulatePipelineRequestBuilder(this, source, xContentType);
     }
 
-    public void allocationExplain(ClusterAllocationExplainRequest request, ActionListener<ClusterAllocationExplainResponse> listener) {
-        execute(TransportClusterAllocationExplainAction.TYPE, request, listener);
-    }
-
-    public ActionFuture<ClusterAllocationExplainResponse> allocationExplain(ClusterAllocationExplainRequest request) {
-        return execute(TransportClusterAllocationExplainAction.TYPE, request);
-    }
-
-    public ClusterAllocationExplainRequestBuilder prepareAllocationExplain() {
-        return new ClusterAllocationExplainRequestBuilder(this);
-    }
-
     public PutStoredScriptRequestBuilder preparePutStoredScript() {
         return new PutStoredScriptRequestBuilder(this);
     }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java
index a3ab6ad8e2f04..c28dcb109b294 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java
@@ -9,9 +9,11 @@
 package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest;
+import org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
@@ -46,19 +48,19 @@ public boolean allowSystemIndexAccessByDefault() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        ClusterAllocationExplainRequest req;
-        if (request.hasContentOrSourceParam() == false) {
-            // Empty request signals "explain the first unassigned shard you find"
-            req = new ClusterAllocationExplainRequest();
-        } else {
+        final var req = new ClusterAllocationExplainRequest(RestUtils.getMasterNodeTimeout(request));
+        if (request.hasContentOrSourceParam()) {
             try (XContentParser parser = request.contentOrSourceParamParser()) {
-                req = ClusterAllocationExplainRequest.parse(parser);
+                ClusterAllocationExplainRequest.parse(req, parser);
             }
-        }
-
+        } // else ok, an empty body means "explain the first unassigned shard you find"
         req.includeYesDecisions(request.paramAsBoolean("include_yes_decisions", false));
         req.includeDiskInfo(request.paramAsBoolean("include_disk_info", false));
-        return channel -> client.admin().cluster().allocationExplain(req, new RestRefCountedChunkedToXContentListener<>(channel));
+        return channel -> client.execute(
+            TransportClusterAllocationExplainAction.TYPE,
+            req,
+            new RestRefCountedChunkedToXContentListener<>(channel)
+        );
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
index f68e83e13496c..f9483bd23f216 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
@@ -133,13 +133,13 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
     public void testFindAnyUnassignedShardToExplain() {
         // find unassigned primary
         ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.UNASSIGNED);
-        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest();
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT);
         ShardRouting shard = findShardToExplain(request, routingAllocation(clusterState));
         assertEquals(clusterState.getRoutingTable().index("idx").shard(0).primaryShard(), shard);
 
         // find unassigned replica
         clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED, ShardRoutingState.UNASSIGNED);
-        request = new ClusterAllocationExplainRequest();
+        request = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT);
         shard = findShardToExplain(request, routingAllocation(clusterState));
         assertEquals(clusterState.getRoutingTable().index("idx").shard(0).replicaShards().get(0), shard);
 
@@ -168,7 +168,7 @@ public void testFindAnyUnassignedShardToExplain() {
             routingTableBuilder.add(indexBuilder);
         }
         clusterState = ClusterState.builder(clusterState).routingTable(routingTableBuilder.build()).build();
-        request = new ClusterAllocationExplainRequest();
+        request = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT);
         shard = findShardToExplain(request, routingAllocation(clusterState));
         assertEquals(clusterState.getRoutingTable().index(redIndex).shard(0).primaryShard(), shard);
 
@@ -179,7 +179,7 @@ public void testFindAnyUnassignedShardToExplain() {
             ShardRoutingState.STARTED,
             ShardRoutingState.STARTED
         );
-        final ClusterAllocationExplainRequest anyUnassignedShardsRequest = new ClusterAllocationExplainRequest();
+        final ClusterAllocationExplainRequest anyUnassignedShardsRequest = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT);
         assertThat(
             expectThrows(
                 IllegalArgumentException.class,
@@ -195,7 +195,7 @@ public void testFindAnyUnassignedShardToExplain() {
 
     public void testFindPrimaryShardToExplain() {
         ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), randomFrom(ShardRoutingState.values()));
-        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest("idx", 0, true, null);
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT, "idx", 0, true, null);
         ShardRouting shard = findShardToExplain(request, routingAllocation(clusterState));
         assertEquals(clusterState.getRoutingTable().index("idx").shard(0).primaryShard(), shard);
     }
@@ -209,7 +209,7 @@ public void testFindAnyReplicaToExplain() {
             ShardRoutingState.STARTED,
             ShardRoutingState.UNASSIGNED
         );
-        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest("idx", 0, false, null);
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT, "idx", 0, false, null);
         ShardRouting shard = findShardToExplain(request, routingAllocation(clusterState));
         assertEquals(
             clusterState.getRoutingTable()
@@ -231,7 +231,7 @@ public void testFindAnyReplicaToExplain() {
             randomFrom(ShardRoutingState.RELOCATING, ShardRoutingState.INITIALIZING),
             ShardRoutingState.STARTED
         );
-        request = new ClusterAllocationExplainRequest("idx", 0, false, null);
+        request = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT, "idx", 0, false, null);
         shard = findShardToExplain(request, routingAllocation(clusterState));
         assertEquals(
             clusterState.getRoutingTable().index("idx").shard(0).replicaShards().stream().filter(ShardRouting::started).findFirst().get(),
@@ -250,7 +250,13 @@ public void testFindShardAssignedToNode() {
         ShardRouting shardToExplain = primary
             ? clusterState.getRoutingTable().index("idx").shard(0).primaryShard()
             : clusterState.getRoutingTable().index("idx").shard(0).replicaShards().get(0);
-        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest("idx", 0, primary, shardToExplain.currentNodeId());
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest(
+            TEST_REQUEST_TIMEOUT,
+            "idx",
+            0,
+            primary,
+            shardToExplain.currentNodeId()
+        );
         RoutingAllocation allocation = routingAllocation(clusterState);
         ShardRouting foundShard = findShardToExplain(request, allocation);
         assertEquals(shardToExplain, foundShard);
@@ -263,7 +269,13 @@ public void testFindShardAssignedToNode() {
                 break;
             }
         }
-        final ClusterAllocationExplainRequest failingRequest = new ClusterAllocationExplainRequest("idx", 0, primary, explainNode);
+        final ClusterAllocationExplainRequest failingRequest = new ClusterAllocationExplainRequest(
+            TEST_REQUEST_TIMEOUT,
+            "idx",
+            0,
+            primary,
+            explainNode
+        );
         expectThrows(IllegalArgumentException.class, () -> findShardToExplain(failingRequest, allocation));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java
index 625a01094792d..cc00207997e9d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java
@@ -15,6 +15,7 @@ public class ClusterAllocationExplainRequestTests extends ESTestCase {
 
     public void testSerialization() throws Exception {
         ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest(
+            randomTimeValue(),
             randomAlphaOfLength(4),
             randomIntBetween(0, Integer.MAX_VALUE),
             randomBoolean(),
@@ -26,6 +27,7 @@ public void testSerialization() throws Exception {
         request.writeTo(output);
 
         ClusterAllocationExplainRequest actual = new ClusterAllocationExplainRequest(output.bytes().streamInput());
+        assertEquals(request.masterNodeTimeout(), actual.masterNodeTimeout());
         assertEquals(request.getIndex(), actual.getIndex());
         assertEquals(request.getShard(), actual.getShard());
         assertEquals(request.isPrimary(), actual.isPrimary());
diff --git a/test/framework/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationUtils.java b/test/framework/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationUtils.java
new file mode 100644
index 0000000000000..b72d95cb02b8f
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationUtils.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.allocation;
+
+import org.elasticsearch.client.internal.ElasticsearchClient;
+
+import static org.elasticsearch.test.ESTestCase.TEST_REQUEST_TIMEOUT;
+import static org.elasticsearch.test.ESTestCase.safeGet;
+
+public class ClusterAllocationExplanationUtils {
+    private ClusterAllocationExplanationUtils() {/* no instances */}
+
+    public static ClusterAllocationExplanation getClusterAllocationExplanation(
+        ElasticsearchClient client,
+        String index,
+        int shard,
+        boolean primary
+    ) {
+        return safeGet(
+            client.execute(
+                TransportClusterAllocationExplainAction.TYPE,
+                new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT, index, shard, primary, null)
+            )
+        ).getExplanation();
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index fb6105005201f..db6fc9ea696d5 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -24,7 +24,9 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
+import org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
@@ -992,7 +994,11 @@ private ClusterHealthStatus ensureColor(
 
             final var detailsFuture = new PlainActionFuture<Void>();
             try (var listeners = new RefCountingListener(detailsFuture)) {
-                clusterAdmin().prepareAllocationExplain().execute(listeners.acquire(allocationExplainRef::set));
+                client().execute(
+                    TransportClusterAllocationExplainAction.TYPE,
+                    new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT),
+                    listeners.acquire(allocationExplainRef::set)
+                );
                 clusterAdmin().prepareState().execute(listeners.acquire(clusterStateRef::set));
                 client().execute(
                     TransportPendingClusterTasksAction.TYPE,
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java
index 50750f629b993..16a2de7bf5b0f 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.ccr;
 
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
+import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -66,14 +66,7 @@ public void testDoNotAllocateFollowerPrimaryToNodesWithoutRemoteClusterClientRol
         final PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, putFollowRequest).get();
         assertFalse(response.isFollowIndexShardsAcked());
         assertFalse(response.isIndexFollowingStarted());
-        final ClusterAllocationExplanation explanation = followerClient().admin()
-            .cluster()
-            .prepareAllocationExplain()
-            .setIndex(followerIndex)
-            .setShard(0)
-            .setPrimary(true)
-            .get()
-            .getExplanation();
+        final var explanation = ClusterAllocationExplanationUtils.getClusterAllocationExplanation(followerClient(), followerIndex, 0, true);
         for (NodeAllocationResult nodeDecision : explanation.getShardAllocationDecision().getAllocateDecision().getNodeDecisions()) {
             assertThat(nodeDecision.getNodeDecision(), equalTo(AllocationDecision.NO));
             if (dataOnlyNodes.contains(nodeDecision.getNode().getName())) {
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java
index 7261ee1f66036..6fe63548a1430 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.cluster.routing.allocation;
 
+import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils;
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction;
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
@@ -541,7 +542,7 @@ private DiscoveryNode getPrimaryShardAssignedNode(int shard) {
 
     private String explainAllocation(int shard) {
         return Strings.toString(
-            clusterAdmin().prepareAllocationExplain().setIndex(index).setShard(shard).setPrimary(true).get().getExplanation(),
+            ClusterAllocationExplanationUtils.getClusterAllocationExplanation(client(), index, shard, true),
             true,
             true
         );
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java
index 628e2c18de2f9..368946d79682c 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.cluster.routing.allocation;
 
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
+import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -110,12 +110,7 @@ public void testIncreaseTierCapacityDiagnosisWhenTierShrinksUnexpectedly() throw
             GetHealthAction.INSTANCE,
             new GetHealthAction.Request(ShardsAvailabilityHealthIndicatorService.NAME, true, 1000)
         ).get();
-        ClusterAllocationExplanation explain = clusterAdmin().prepareAllocationExplain()
-            .setIndex("test")
-            .setShard(0)
-            .setPrimary(false)
-            .get()
-            .getExplanation();
+        final var explain = ClusterAllocationExplanationUtils.getClusterAllocationExplanation(client(), "test", 0, false);
         logger.info(XContentHelper.toXContent(explain, XContentType.JSON, true).utf8ToString());
         HealthIndicatorResult indicatorResult = healthResponse.findIndicator(ShardsAvailabilityHealthIndicatorService.NAME);
         assertThat(indicatorResult.status(), equalTo(HealthStatus.YELLOW));
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java
index 01288b4135d15..7a0e00e5c4147 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java
@@ -7,8 +7,7 @@
 
 package org.elasticsearch.xpack.ilm;
 
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest;
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
+import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -222,10 +221,9 @@ public void testUserOptsOutOfTierMigration() throws Exception {
     }
 
     private void assertReplicaIsUnassigned() {
-        ClusterAllocationExplainRequest explainReplicaShard = new ClusterAllocationExplainRequest().setIndex(managedIndex)
-            .setPrimary(false)
-            .setShard(0);
-        ClusterAllocationExplainResponse response = clusterAdmin().allocationExplain(explainReplicaShard).actionGet();
-        assertThat(response.getExplanation().getShardState(), is(ShardRoutingState.UNASSIGNED));
+        assertThat(
+            ClusterAllocationExplanationUtils.getClusterAllocationExplanation(client(), managedIndex, 0, false).getShardState(),
+            is(ShardRoutingState.UNASSIGNED)
+        );
     }
 }
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java
index 4b9e1b0d9211e..be37cc2184d15 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java
@@ -12,6 +12,8 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FilterDirectory;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest;
+import org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStatus;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@@ -334,13 +336,11 @@ public void testCreateAndRestorePartialSearchableSnapshot() throws Exception {
         assertThat(indicesAdmin().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1));
         assertTotalHits(aliasName, originalAllHits, originalBarHits);
 
-        final Decision diskDeciderDecision = clusterAdmin().prepareAllocationExplain()
-            .setIndex(restoredIndexName)
+        final var request = new ClusterAllocationExplainRequest(TEST_REQUEST_TIMEOUT).setIndex(restoredIndexName)
             .setShard(0)
-            .setPrimary(true)
-            .setIncludeYesDecisions(true)
-            .get()
-            .getExplanation()
+            .setPrimary(true);
+        request.includeYesDecisions(true);
+        final var diskDeciderDecision = safeGet(client().execute(TransportClusterAllocationExplainAction.TYPE, request)).getExplanation()
             .getShardAllocationDecision()
             .getMoveDecision()
             .getCanRemainDecision()
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
index 8fc95ad4491e8..50be307fe16a4 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction;
@@ -80,6 +79,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils.getClusterAllocationExplanation;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING;
 import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
@@ -972,13 +972,7 @@ public void testSnapshotOfSearchableSnapshotCanBeRestoredBeforeRepositoryRegiste
         );
 
         assertBusy(() -> {
-            final ClusterAllocationExplanation clusterAllocationExplanation = clusterAdmin().prepareAllocationExplain()
-                .setIndex(restoredIndexName)
-                .setShard(0)
-                .setPrimary(true)
-                .get()
-                .getExplanation();
-
+            final var clusterAllocationExplanation = getClusterAllocationExplanation(client(), restoredIndexName, 0, true);
             final String description = Strings.toString(clusterAllocationExplanation);
             final AllocateUnassignedDecision allocateDecision = clusterAllocationExplanation.getShardAllocationDecision()
                 .getAllocateDecision();
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java
index a55521394f548..28da64b4ceaec 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.support.ActionTestUtils;
 import org.elasticsearch.action.support.SubscribableListener;
@@ -49,6 +48,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils.getClusterAllocationExplanation;
 import static org.elasticsearch.blobcache.shared.SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.DataTier.TIER_PREFERENCE;
 import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING;
@@ -109,12 +109,7 @@ public void testPartialSearchableSnapshotNotAllocatedToNodesWithoutCache() throw
         final ClusterState state = clusterAdmin().prepareState().clear().setRoutingTable(true).get().getState();
         assertTrue(state.toString(), state.routingTable().index(req.mountedIndexName()).allPrimaryShardsUnassigned());
 
-        final ClusterAllocationExplanation explanation = clusterAdmin().prepareAllocationExplain()
-            .setPrimary(true)
-            .setIndex(req.mountedIndexName())
-            .setShard(0)
-            .get()
-            .getExplanation();
+        final var explanation = getClusterAllocationExplanation(client(), req.mountedIndexName(), 0, true);
         for (NodeAllocationResult nodeDecision : explanation.getShardAllocationDecision().getAllocateDecision().getNodeDecisions()) {
             assertTrue(
                 nodeDecision.getNode() + " vs " + Strings.toString(explanation),
@@ -232,13 +227,7 @@ public void testPartialSearchableSnapshotDelaysAllocationUntilNodeCacheStatesKno
 
         assertBusy(() -> {
             try {
-                final ClusterAllocationExplanation explanation = clusterAdmin().prepareAllocationExplain()
-                    .setPrimary(true)
-                    .setIndex(req.mountedIndexName())
-                    .setShard(0)
-                    .get()
-                    .getExplanation();
-
+                final var explanation = getClusterAllocationExplanation(client(), req.mountedIndexName(), 0, true);
                 assertTrue(Strings.toString(explanation), explanation.getShardAllocationDecision().getAllocateDecision().isDecisionTaken());
 
                 assertThat(
@@ -257,13 +246,7 @@ public void testPartialSearchableSnapshotDelaysAllocationUntilNodeCacheStatesKno
 
         // Still won't be allocated
         assertFalse(responseFuture.isDone());
-        final ClusterAllocationExplanation explanation = clusterAdmin().prepareAllocationExplain()
-            .setPrimary(true)
-            .setIndex(req.mountedIndexName())
-            .setShard(0)
-            .get()
-            .getExplanation();
-
+        final var explanation = getClusterAllocationExplanation(client(), req.mountedIndexName(), 0, true);
         assertThat(
             Strings.toString(explanation),
             explanation.getShardAllocationDecision().getAllocateDecision().getAllocationStatus(),
diff --git a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java
index 0c70fd5a252c4..1594f78e04140 100644
--- a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java
+++ b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownShardsIT.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.shutdown;
 
-import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
+import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.cluster.ClusterState;
@@ -148,15 +148,8 @@ public void testNodeReplacementOnlyAllowsShardsFromReplacedNode() throws Excepti
 
         ensureYellow("other");
 
-        // Explain the replica for the "other" index
-        ClusterAllocationExplainResponse explainResponse = clusterAdmin().prepareAllocationExplain()
-            .setIndex("other")
-            .setShard(0)
-            .setPrimary(false)
-            .get();
-
         // Validate that the replica cannot be allocated to nodeB because it's the target of a node replacement
-        explainResponse.getExplanation()
+        ClusterAllocationExplanationUtils.getClusterAllocationExplanation(client(), "other", 0, false)
             .getShardAllocationDecision()
             .getAllocateDecision()
             .getNodeDecisions()
@@ -209,15 +202,8 @@ public void testNodeReplacementOverridesFilters() throws Exception {
 
         ensureYellow("other");
 
-        // Explain the replica for the "other" index
-        ClusterAllocationExplainResponse explainResponse = clusterAdmin().prepareAllocationExplain()
-            .setIndex("other")
-            .setShard(0)
-            .setPrimary(false)
-            .get();
-
         // Validate that the replica cannot be allocated to nodeB because it's the target of a node replacement
-        explainResponse.getExplanation()
+        ClusterAllocationExplanationUtils.getClusterAllocationExplanation(client(), "other", 0, false)
             .getShardAllocationDecision()
             .getAllocateDecision()
             .getNodeDecisions()

From 3f23c151803a5fe057ed0c2f484445b060b178ef Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 31 May 2024 13:04:57 +0100
Subject: [PATCH 076/208] Fix misc trappy allocation API timeouts (#109241)

Get/delete desired balance and get-allocation-stats APIs

Relates #107984
---
 docs/changelog/109241.yaml                             |  5 +++++
 .../api/_internal.delete_desired_balance.json          |  6 ++++++
 .../api/_internal.get_desired_balance.json             |  6 ++++++
 .../allocation/TransportGetDesiredBalanceActionIT.java | 10 ++++++----
 .../cluster/allocation/DesiredBalanceRequest.java      |  5 +++--
 .../allocation/TransportGetAllocationStatsAction.java  |  5 +++--
 .../cluster/node/stats/TransportNodesStatsAction.java  |  7 ++++++-
 .../admin/cluster/RestDeleteDesiredBalanceAction.java  |  8 +++-----
 .../admin/cluster/RestGetDesiredBalanceAction.java     |  4 +++-
 .../TransportDeleteDesiredBalanceActionTests.java      |  4 ++--
 .../TransportGetDesiredBalanceActionTests.java         |  2 +-
 11 files changed, 44 insertions(+), 18 deletions(-)
 create mode 100644 docs/changelog/109241.yaml

diff --git a/docs/changelog/109241.yaml b/docs/changelog/109241.yaml
new file mode 100644
index 0000000000000..b7343b9df1841
--- /dev/null
+++ b/docs/changelog/109241.yaml
@@ -0,0 +1,5 @@
+pr: 109241
+summary: Fix misc trappy allocation API timeouts
+area: Allocation
+type: bug
+issues: []
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.delete_desired_balance.json b/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.delete_desired_balance.json
index d95efed82052b..b5c4eaa7fc6f9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.delete_desired_balance.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.delete_desired_balance.json
@@ -18,6 +18,12 @@
           ]
         }
       ]
+    },
+    "params": {
+      "master_timeout":{
+        "type":"time",
+        "description":"Timeout for connection to master node"
+      }
     }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.get_desired_balance.json b/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.get_desired_balance.json
index aa6285b1f9d1d..5922d77d99dc0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.get_desired_balance.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.get_desired_balance.json
@@ -18,6 +18,12 @@
           ]
         }
       ]
+    },
+    "params": {
+      "master_timeout":{
+        "type":"time",
+        "description":"Timeout for connection to master node"
+      }
     }
   }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java
index a4cf7843beb41..d0e0543bcca03 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java
@@ -40,8 +40,9 @@ public void testDesiredBalanceOnMultiNodeCluster() throws Exception {
         var clusterHealthResponse = clusterAdmin().health(new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)).get();
         assertEquals(RestStatus.OK, clusterHealthResponse.status());
 
-        DesiredBalanceResponse desiredBalanceResponse = client().execute(TransportGetDesiredBalanceAction.TYPE, new DesiredBalanceRequest())
-            .get();
+        final var desiredBalanceResponse = safeGet(
+            client().execute(TransportGetDesiredBalanceAction.TYPE, new DesiredBalanceRequest(TEST_REQUEST_TIMEOUT))
+        );
 
         assertEquals(1, desiredBalanceResponse.getRoutingTable().size());
         Map<Integer, DesiredBalanceResponse.DesiredShards> shardsMap = desiredBalanceResponse.getRoutingTable().get(index);
@@ -75,8 +76,9 @@ public void testDesiredBalanceWithUnassignedShards() throws Exception {
         var clusterHealthResponse = clusterAdmin().health(new ClusterHealthRequest(index).waitForStatus(ClusterHealthStatus.YELLOW)).get();
         assertEquals(RestStatus.OK, clusterHealthResponse.status());
 
-        DesiredBalanceResponse desiredBalanceResponse = client().execute(TransportGetDesiredBalanceAction.TYPE, new DesiredBalanceRequest())
-            .get();
+        final var desiredBalanceResponse = safeGet(
+            client().execute(TransportGetDesiredBalanceAction.TYPE, new DesiredBalanceRequest(TEST_REQUEST_TIMEOUT))
+        );
 
         assertEquals(1, desiredBalanceResponse.getRoutingTable().size());
         Map<Integer, DesiredBalanceResponse.DesiredShards> shardsMap = desiredBalanceResponse.getRoutingTable().get(index);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceRequest.java
index b34ba313894b9..d01c05b63ab67 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceRequest.java
@@ -10,12 +10,13 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
 public class DesiredBalanceRequest extends MasterNodeReadRequest<DesiredBalanceRequest> {
-    public DesiredBalanceRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public DesiredBalanceRequest(TimeValue masterNodeTimeout) {
+        super(masterNodeTimeout);
     }
 
     public DesiredBalanceRequest(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
index f26921fd47260..560ef6feae1e4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -102,8 +103,8 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state)
 
     public static class Request extends MasterNodeReadRequest<Request> {
 
-        public Request(TaskId parentTaskId) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        public Request(TimeValue masterNodeTimeout, TaskId parentTaskId) {
+            super(masterNodeTimeout);
             setParentTask(parentTaskId);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index 4829f309e1f30..a84905e7f4c8e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.node.NodeService;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -37,6 +38,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 public class TransportNodesStatsAction extends TransportNodesAction<
@@ -89,7 +91,10 @@ protected void newResponseAsync(
             || NodesStatsRequestParameters.Metric.FS.containedIn(metrics)) {
             client.execute(
                 TransportGetAllocationStatsAction.TYPE,
-                new TransportGetAllocationStatsAction.Request(new TaskId(clusterService.localNode().getId(), task.getId())),
+                new TransportGetAllocationStatsAction.Request(
+                    Objects.requireNonNullElse(request.timeout(), RestUtils.REST_MASTER_TIMEOUT_DEFAULT),
+                    new TaskId(clusterService.localNode().getId(), task.getId())
+                ),
                 listener.delegateFailure((l, r) -> {
                     ActionListener.respondAndRelease(
                         l,
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteDesiredBalanceAction.java
index f0b516a876622..6a42e9a2a3f40 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteDesiredBalanceAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteDesiredBalanceAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -35,10 +36,7 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        return channel -> client.execute(
-            TransportDeleteDesiredBalanceAction.TYPE,
-            new DesiredBalanceRequest(),
-            new RestToXContentListener<>(channel)
-        );
+        final var req = new DesiredBalanceRequest(RestUtils.getMasterNodeTimeout(request));
+        return channel -> client.execute(TransportDeleteDesiredBalanceAction.TYPE, req, new RestToXContentListener<>(channel));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetDesiredBalanceAction.java
index 0bb7cc5ff7473..cb7b3780be234 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetDesiredBalanceAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetDesiredBalanceAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
@@ -35,9 +36,10 @@ public List<Route> routes() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        final var req = new DesiredBalanceRequest(RestUtils.getMasterNodeTimeout(request));
         return restChannel -> client.execute(
             TransportGetDesiredBalanceAction.TYPE,
-            new DesiredBalanceRequest(),
+            req,
             new RestRefCountedChunkedToXContentListener<>(restChannel)
         );
     }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java
index 6eb3310623b92..6885e6851c77d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportDeleteDesiredBalanceActionTests.java
@@ -76,7 +76,7 @@ public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception {
             mock(IndexNameExpressionResolver.class),
             mock(AllocationService.class),
             mock(ShardsAllocator.class)
-        ).masterOperation(mock(Task.class), new DesiredBalanceRequest(), ClusterState.EMPTY_STATE, listener);
+        ).masterOperation(mock(Task.class), new DesiredBalanceRequest(TEST_REQUEST_TIMEOUT), ClusterState.EMPTY_STATE, listener);
 
         var exception = expectThrows(ResourceNotFoundException.class, listener);
         assertThat(exception.getMessage(), equalTo("Desired balance allocator is not in use, no desired balance found"));
@@ -156,7 +156,7 @@ public DesiredBalance compute(
             allocator
         );
 
-        action.masterOperation(mock(Task.class), new DesiredBalanceRequest(), clusterState, listener);
+        action.masterOperation(mock(Task.class), new DesiredBalanceRequest(TEST_REQUEST_TIMEOUT), clusterState, listener);
 
         try {
             assertThat(listener.get(), notNullValue());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java
index 69cd9b4026108..414dc45ee458f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java
@@ -91,7 +91,7 @@ private static DesiredBalanceResponse execute(TransportGetDesiredBalanceAction a
         return PlainActionFuture.get(
             future -> action.masterOperation(
                 new Task(1, "test", TransportGetDesiredBalanceAction.TYPE.name(), "", TaskId.EMPTY_TASK_ID, Map.of()),
-                new DesiredBalanceRequest(),
+                new DesiredBalanceRequest(TEST_REQUEST_TIMEOUT),
                 clusterState,
                 future
             ),

From 92f89357b8796a331acef8150a659a705f351ceb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20Fred=C3=A9n?=
 <109296772+jfreden@users.noreply.github.com>
Date: Fri, 31 May 2024 14:24:38 +0200
Subject: [PATCH 077/208] Set API key version to >= current node version
 (#109231)

---
 .../elasticsearch/xpack/core/security/action/apikey/ApiKey.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java
index cee63c16229e0..c51d897912fb5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java
@@ -103,7 +103,7 @@ public VersionId<?> versionNumber() {
         }
     }
 
-    public static final ApiKey.Version CURRENT_API_KEY_VERSION = new ApiKey.Version(8_13_00_99);
+    public static final ApiKey.Version CURRENT_API_KEY_VERSION = new ApiKey.Version(8_15_00_99);
 
     private final String name;
     private final String id;

From f416688e1eb2e021263421dd01cdbae3fe4e8535 Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 31 May 2024 13:36:40 +0100
Subject: [PATCH 078/208] Fix double-pausing shard snapshot (#109148)

Closes #109143
---
 docs/changelog/109148.yaml                    |  6 ++
 .../snapshots/SnapshotsService.java           |  9 +++
 .../snapshots/SnapshotsServiceTests.java      | 64 +++++++++++++++++++
 3 files changed, 79 insertions(+)
 create mode 100644 docs/changelog/109148.yaml

diff --git a/docs/changelog/109148.yaml b/docs/changelog/109148.yaml
new file mode 100644
index 0000000000000..902da6f1a1db3
--- /dev/null
+++ b/docs/changelog/109148.yaml
@@ -0,0 +1,6 @@
+pr: 109148
+summary: Fix double-pausing shard snapshot
+area: Snapshot/Restore
+type: bug
+issues:
+ - 109143
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index dd8ddcffd5fe3..b4c15e7b77b78 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -3354,6 +3354,15 @@ private <T> void executeShardSnapshotUpdate(
                     updatedState = updateSnapshotState.updatedState;
                 }
 
+                if (updatedState.state() == ShardState.PAUSED_FOR_NODE_REMOVAL) {
+                    // leave subsequent entries for this shard alone until this one is unpaused
+                    iterator.remove();
+                } else {
+                    // All other shard updates leave the shard in a complete state, which means we should leave this update in the list so
+                    // it can fall through to later entries and start any waiting shard snapshots:
+                    assert updatedState.isActive() == false : updatedState;
+                }
+
                 logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshot, updatedShard, updatedState.state());
                 changedCount++;
                 newStates.get().put(updatedShard, updatedState);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java
index 56a28b11edfe7..bcc7a23bbec53 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java
@@ -401,6 +401,70 @@ public void testCompletedCloneStartsNextClone() throws Exception {
         assertIsNoop(updatedClusterState, completeShardClone);
     }
 
+    public void testPauseForNodeRemovalWithQueuedShards() throws Exception {
+        final var repoName = "test-repo";
+        final var snapshot1 = snapshot(repoName, "snap-1");
+        final var snapshot2 = snapshot(repoName, "snap-2");
+        final var indexName = "index-1";
+        final var shardId = new ShardId(index(indexName), 0);
+        final var repositoryShardId = new RepositoryShardId(indexId(indexName), 0);
+        final var nodeId = uuid();
+
+        final var runningEntry = snapshotEntry(
+            snapshot1,
+            Collections.singletonMap(indexName, repositoryShardId.index()),
+            Map.of(shardId, initShardStatus(nodeId))
+        );
+
+        final var queuedEntry = snapshotEntry(
+            snapshot2,
+            Collections.singletonMap(indexName, repositoryShardId.index()),
+            Map.of(shardId, SnapshotsInProgress.ShardSnapshotStatus.UNASSIGNED_QUEUED)
+        );
+
+        final var initialState = stateWithSnapshots(
+            ClusterState.builder(ClusterState.EMPTY_STATE)
+                .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create(nodeId)).localNodeId(nodeId).masterNodeId(nodeId).build())
+                .routingTable(
+                    RoutingTable.builder()
+                        .add(
+                            IndexRoutingTable.builder(shardId.getIndex())
+                                .addShard(TestShardRouting.newShardRouting(shardId, nodeId, true, ShardRoutingState.STARTED))
+                        )
+                        .build()
+                )
+                .build(),
+            repoName,
+            runningEntry,
+            queuedEntry
+        );
+
+        final var updatedState = applyUpdates(
+            initialState,
+            new SnapshotsService.ShardSnapshotUpdate(
+                snapshot1,
+                shardId,
+                null,
+                new SnapshotsInProgress.ShardSnapshotStatus(
+                    nodeId,
+                    SnapshotsInProgress.ShardState.PAUSED_FOR_NODE_REMOVAL,
+                    runningEntry.shards().get(shardId).generation()
+                ),
+                ActionTestUtils.assertNoFailureListener(t -> {})
+            )
+        );
+
+        assertEquals(
+            SnapshotsInProgress.ShardState.PAUSED_FOR_NODE_REMOVAL,
+            SnapshotsInProgress.get(updatedState).snapshot(snapshot1).shards().get(shardId).state()
+        );
+
+        assertEquals(
+            SnapshotsInProgress.ShardState.QUEUED,
+            SnapshotsInProgress.get(updatedState).snapshot(snapshot2).shards().get(shardId).state()
+        );
+    }
+
     public void testSnapshottingIndicesExcludesClones() {
         final String repoName = "test-repo";
         final String indexName = "index";

From 59592ece0b99c01a0b88315245606d17b7094a71 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 31 May 2024 09:08:02 -0400
Subject: [PATCH 079/208] ESQL: More javadoc for SurrogateExpression (#109218)

---
 .../esql/expression/SurrogateExpression.java      | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/SurrogateExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/SurrogateExpression.java
index 590b68a6892e5..b28d7a1036def 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/SurrogateExpression.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/SurrogateExpression.java
@@ -8,15 +8,24 @@
 package org.elasticsearch.xpack.esql.expression;
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 
 /**
  * Interface signaling to the planner that the declaring expression
- * has to be replaced by a different form: e.g. avg = sum / count
+ * has to be replaced by a different form.
+ * Implement this on {@link AggregateFunction}s when either:
+ * <ul>
+ *     <li>The aggregation doesn't have a "native" implementation and instead
+ *     should be replaced with a combination of aggregations and then
+ *     "put back together" on output. Like {@code AVG = SUM / COUNT}.</li>
+ *     <li>The aggregation is folded if it receives constant
+ *     input. Like {@code MIN(1) == 1}.</li>
+ * </ul>
  */
 public interface SurrogateExpression {
-
     /**
-     * Returns the expression to be replaced by or {@code null} if this cannot be replaced.
+     * Returns the expression to be replaced by or {@code null} if this cannot
+     * be replaced.
      */
     Expression surrogate();
 }

From f7a20ddf4192434b079403514a34af0d19207510 Mon Sep 17 00:00:00 2001
From: Yang Wang <yang.wang@elastic.co>
Date: Fri, 31 May 2024 23:13:02 +1000
Subject: [PATCH 080/208] Handle nullable DocsStats and StoresStats (#109196)

Both DocsStats and StoresStats from CommonStats can be null. This PR
adds necessary checks for nullity in a few places.
---
 docs/changelog/109196.yaml                                 | 5 +++++
 .../admin/indices/rollover/TransportRolloverAction.java    | 4 ++--
 .../cluster/MockInternalClusterInfoService.java            | 2 +-
 .../core/datatiers/NodesDataTiersUsageTransportAction.java | 7 +++++--
 .../application/EnterpriseSearchUsageTransportAction.java  | 5 ++++-
 .../xpack/ml/job/retention/EmptyStateIndexRemover.java     | 2 +-
 6 files changed, 18 insertions(+), 7 deletions(-)
 create mode 100644 docs/changelog/109196.yaml

diff --git a/docs/changelog/109196.yaml b/docs/changelog/109196.yaml
new file mode 100644
index 0000000000000..7f5ca3efbc8d4
--- /dev/null
+++ b/docs/changelog/109196.yaml
@@ -0,0 +1,5 @@
+pr: 109196
+summary: Handle nullable `DocsStats` and `StoresStats`
+area: Distributed
+type: bug
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index abf42cffdaa01..bf059f6fe868e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -366,7 +366,7 @@ static Condition.Stats buildStats(@Nullable final IndexMetadata metadata, @Nulla
                 .flatMap(Arrays::stream)
                 .filter(shard -> shard.getShardRouting().primary())
                 .map(ShardStats::getStats)
-                .mapToLong(shard -> shard.docs.getTotalSizeInBytes())
+                .mapToLong(shard -> shard.docs == null ? 0L : shard.docs.getTotalSizeInBytes())
                 .max()
                 .orElse(0);
 
@@ -376,7 +376,7 @@ static Condition.Stats buildStats(@Nullable final IndexMetadata metadata, @Nulla
                 .flatMap(Arrays::stream)
                 .filter(shard -> shard.getShardRouting().primary())
                 .map(ShardStats::getStats)
-                .mapToLong(shard -> shard.docs.getCount())
+                .mapToLong(shard -> shard.docs == null ? 0L : shard.docs.getCount())
                 .max()
                 .orElse(0);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
index e07c27b22c926..6b9e884d41e2e 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
@@ -108,7 +108,7 @@ ShardStats[] adjustShardStats(ShardStats[] shardsStats) {
             var storeStats = new StoreStats(
                 shardSizeFunctionCopy.apply(shardRouting),
                 shardSizeFunctionCopy.apply(shardRouting),
-                shardStats.getStats().store.reservedSizeInBytes()
+                shardStats.getStats().store == null ? 0L : shardStats.getStats().store.reservedSizeInBytes()
             );
             var commonStats = new CommonStats(new CommonStatsFlags(CommonStatsFlags.Flag.Store));
             commonStats.store = storeStats;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java
index ee721d9d55714..c20d97b7125a0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.NodeIndicesStats;
@@ -137,8 +138,10 @@ static Map<String, NodeDataTiersUsage.UsageStats> aggregateStats(
                 List<IndexShardStats> allShardStats = nodeIndicesStats.getShardStats(indexMetadata.getIndex());
                 if (allShardStats != null) {
                     for (IndexShardStats indexShardStats : allShardStats) {
-                        usageStats.incrementTotalSize(indexShardStats.getTotal().getStore().totalDataSetSizeInBytes());
-                        usageStats.incrementDocCount(indexShardStats.getTotal().getDocs().getCount());
+                        final StoreStats storeStats = indexShardStats.getTotal().getStore();
+                        usageStats.incrementTotalSize(storeStats == null ? 0L : storeStats.totalDataSetSizeInBytes());
+                        final DocsStats docsStats = indexShardStats.getTotal().getDocs();
+                        usageStats.incrementDocCount(docsStats == null ? 0L : docsStats.getCount());
 
                         ShardRouting shardRouting = routingNode.getByShardId(indexShardStats.getShardId());
                         if (shardRouting != null && shardRouting.state() == ShardRoutingState.STARTED) {
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java
index 73fba38a2022f..d9b7c325dfd21 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java
@@ -47,6 +47,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN;
 import static org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage.MAX_RULE_COUNT;
@@ -175,7 +176,9 @@ public void onResponse(IndicesStatsResponse indicesStatsResponse) {
                     Map<String, IndexStats> indicesStats = indicesStatsResponse.getIndices();
                     int queryRulesetCount = indicesStats.values()
                         .stream()
-                        .mapToInt(indexShardStats -> (int) indexShardStats.getPrimaries().getDocs().getCount())
+                        .map(indexShardStats -> indexShardStats.getPrimaries().getDocs())
+                        .filter(Objects::nonNull)
+                        .mapToInt(docsStats -> (int) docsStats.getCount())
                         .sum();
 
                     ListQueryRulesetsAction.Request queryRulesetsCountRequest = new ListQueryRulesetsAction.Request(
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemover.java
index 1c8c100939dc7..337b4c397493e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/EmptyStateIndexRemover.java
@@ -73,7 +73,7 @@ private void getEmptyStateIndices(ActionListener<Set<String>> listener) {
                         indicesStatsResponse.getIndices()
                             .values()
                             .stream()
-                            .filter(stats -> stats.getTotal().getDocs().getCount() == 0)
+                            .filter(stats -> stats.getTotal().getDocs() == null || stats.getTotal().getDocs().getCount() == 0)
                             .map(IndexStats::getIndex)
                             .collect(toSet())
                     )

From 5bd1bfc4255ce4068f834b52858c32fcee1d3a5f Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 31 May 2024 09:50:01 -0400
Subject: [PATCH 081/208] ESQL: Move serialization for EsField (#109222)

This moves the serialization logic for `EsField` into the `EsField`
subclasses to better align with the way rest of Elasticsearch works. It
also switches them from ESQL's home grown `writeNamed` thing to
`NamedWriteable`. These are wire compatible with one another.
---
 .../xpack/esql/core/type/DateEsField.java     |  24 +++-
 .../xpack/esql/core/type/EsField.java         |  44 ++++++-
 .../esql/core/type/InvalidMappedField.java    |  29 ++++-
 .../xpack/esql/core/type/KeywordEsField.java  |  39 +++++-
 .../xpack/esql/core/type/TextEsField.java     |  24 +++-
 .../esql/core/type/UnsupportedEsField.java    |  31 ++++-
 .../core/type/AbstractEsFieldTypeTests.java   |  65 +++++++++
 .../esql/core/type/DateEsFieldTests.java      |  36 +++++
 .../xpack/esql/core/type/EsFieldTests.java    |  45 +++++++
 .../core/type/InvalidMappedFieldTests.java    |  39 ++++++
 .../esql/core/type/KeywordEsFieldTests.java   |  50 +++++++
 .../esql/core/type/TextEsFieldTests.java      |  42 ++++++
 .../core/type/UnsupportedEsFieldTests.java    |  42 ++++++
 .../xpack/esql/io/stream/PlanNamedTypes.java  | 123 +-----------------
 .../xpack/esql/io/stream/PlanStreamInput.java |   5 -
 .../xpack/esql/plugin/EsqlPlugin.java         |  42 +++---
 .../xpack/esql/SerializationTestUtils.java    |  27 ++--
 .../esql/io/stream/PlanNamedTypesTests.java   |  66 ----------
 .../esql/plugin/DataNodeRequestTests.java     |   1 +
 19 files changed, 543 insertions(+), 231 deletions(-)
 create mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java
 create mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java
 create mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
 create mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java
 create mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java
 create mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java
 create mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
index a8c5ca7f12807..9d2341f77505e 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
@@ -6,12 +6,18 @@
  */
 package org.elasticsearch.xpack.esql.core.type;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
 import java.util.Map;
 
 /**
- * SQL-related information about an index field with date type
+ * Information about a field in an ES index with the {@code date} type
  */
 public class DateEsField extends EsField {
+    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(EsField.class, "DateEsField", DateEsField::new);
 
     public static DateEsField dateEsField(String name, Map<String, EsField> properties, boolean hasDocValues) {
         return new DateEsField(name, DataTypes.DATETIME, properties, hasDocValues);
@@ -20,4 +26,20 @@ public static DateEsField dateEsField(String name, Map<String, EsField> properti
     private DateEsField(String name, DataType dataType, Map<String, EsField> properties, boolean hasDocValues) {
         super(name, dataType, properties, hasDocValues);
     }
+
+    private DateEsField(StreamInput in) throws IOException {
+        this(in.readString(), DataTypes.DATETIME, in.readMap(i -> i.readNamedWriteable(EsField.class)), in.readBoolean());
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(getName());
+        out.writeMap(getProperties(), StreamOutput::writeNamedWriteable);
+        out.writeBoolean(isAggregatable());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java
index 6cb5c97545fdb..bdc60ebab55ef 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java
@@ -6,15 +6,33 @@
  */
 package org.elasticsearch.xpack.esql.core.type;
 
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 
+import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
 /**
- * SQL-related information about an index field
+ * Information about a field in an ES index.
  */
-public class EsField {
+public class EsField implements NamedWriteable {
+    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        return List.of(
+            EsField.ENTRY,
+            DateEsField.ENTRY,
+            InvalidMappedField.ENTRY,
+            KeywordEsField.ENTRY,
+            TextEsField.ENTRY,
+            UnsupportedEsField.ENTRY
+        );
+    }
+
+    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(EsField.class, "EsField", EsField::new);
 
     private final DataType esDataType;
     private final boolean aggregatable;
@@ -34,6 +52,28 @@ public EsField(String name, DataType esDataType, Map<String, EsField> properties
         this.isAlias = isAlias;
     }
 
+    public EsField(StreamInput in) throws IOException {
+        this.name = in.readString();
+        this.esDataType = DataType.readFrom(in);
+        this.properties = in.readImmutableMap(StreamInput::readString, i -> i.readNamedWriteable(EsField.class));
+        this.aggregatable = in.readBoolean();
+        this.isAlias = in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeString(esDataType.typeName());
+        out.writeMap(properties, StreamOutput::writeNamedWriteable);
+        out.writeBoolean(aggregatable);
+        out.writeBoolean(isAlias);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     /**
      * Returns the field path
      */
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
index 47f883c9ff1d0..3f9a1415df70c 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
@@ -7,8 +7,12 @@
 
 package org.elasticsearch.xpack.esql.core.type;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 
+import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
 import java.util.TreeMap;
@@ -18,6 +22,11 @@
  * Used during mapping discovery only.
  */
 public class InvalidMappedField extends EsField {
+    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        EsField.class,
+        "InvalidMappedField",
+        InvalidMappedField::new
+    );
 
     private final String errorMessage;
 
@@ -27,11 +36,27 @@ public InvalidMappedField(String name, String errorMessage, Map<String, EsField>
     }
 
     public InvalidMappedField(String name, String errorMessage) {
-        this(name, errorMessage, new TreeMap<String, EsField>());
+        this(name, errorMessage, new TreeMap<>());
     }
 
     public InvalidMappedField(String name) {
-        this(name, StringUtils.EMPTY, new TreeMap<String, EsField>());
+        this(name, StringUtils.EMPTY, new TreeMap<>());
+    }
+
+    private InvalidMappedField(StreamInput in) throws IOException {
+        this(in.readString(), in.readString(), in.readImmutableMap(StreamInput::readString, i -> i.readNamedWriteable(EsField.class)));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(getName());
+        out.writeString(errorMessage);
+        out.writeMap(getProperties(), StreamOutput::writeNamedWriteable);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
     }
 
     public String errorMessage() {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
index f041890ed0e51..975bd75f91f18 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
@@ -6,6 +6,11 @@
  */
 package org.elasticsearch.xpack.esql.core.type;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
@@ -13,9 +18,14 @@
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
 
 /**
- * SQL-related information about an index field with keyword type
+ * Information about a field in an ES index with the {@code keyword} type.
  */
 public class KeywordEsField extends EsField {
+    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        EsField.class,
+        "KeywordEsField",
+        KeywordEsField::new
+    );
 
     private final int precision;
     private final boolean normalized;
@@ -53,6 +63,33 @@ protected KeywordEsField(
         this.normalized = normalized;
     }
 
+    private KeywordEsField(StreamInput in) throws IOException {
+        this(
+            in.readString(),
+            KEYWORD,
+            in.readMap(i -> i.readNamedWriteable(EsField.class)),
+            in.readBoolean(),
+            in.readInt(),
+            in.readBoolean(),
+            in.readBoolean()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(getName());
+        out.writeMap(getProperties(), StreamOutput::writeNamedWriteable);
+        out.writeBoolean(isAggregatable());
+        out.writeInt(precision);
+        out.writeBoolean(normalized);
+        out.writeBoolean(isAlias());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     public int getPrecision() {
         return precision;
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
index 38b78a2714c1a..ba5295f1f9a8a 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
@@ -6,9 +6,13 @@
  */
 package org.elasticsearch.xpack.esql.core.type;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 
+import java.io.IOException;
 import java.util.Map;
 import java.util.function.Function;
 
@@ -16,9 +20,10 @@
 import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
 
 /**
- * SQL-related information about an index field with text type
+ * Information about a field in an es index with the {@code text} type.
  */
 public class TextEsField extends EsField {
+    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(EsField.class, "TextEsField", TextEsField::new);
 
     public TextEsField(String name, Map<String, EsField> properties, boolean hasDocValues) {
         this(name, properties, hasDocValues, false);
@@ -28,6 +33,23 @@ public TextEsField(String name, Map<String, EsField> properties, boolean hasDocV
         super(name, TEXT, properties, hasDocValues, isAlias);
     }
 
+    private TextEsField(StreamInput in) throws IOException {
+        this(in.readString(), in.readMap(i -> i.readNamedWriteable(EsField.class)), in.readBoolean(), in.readBoolean());
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(getName());
+        out.writeMap(getProperties(), StreamOutput::writeNamedWriteable);
+        out.writeBoolean(isAggregatable());
+        out.writeBoolean(isAlias());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     public EsField getExactField() {
         Tuple<EsField, String> findExact = findExact();
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
index 1eb558020d232..af1d0d3659bc0 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
@@ -6,15 +6,25 @@
  */
 package org.elasticsearch.xpack.esql.core.type;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
 import java.util.TreeMap;
 
 /**
- * SQL-related information about an index field that cannot be supported by SQL.
- * All the subfields (properties) of an unsupported type should also be unsupported.
+ * Information about a field in an ES index that cannot be supported by ESQL.
+ * All the subfields (properties) of an unsupported type are also be unsupported.
  */
 public class UnsupportedEsField extends EsField {
+    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        EsField.class,
+        "UnsupportedEsField",
+        UnsupportedEsField::new
+    );
 
     private final String originalType;
     private final String inherited; // for fields belonging to parents (or grandparents) that have an unsupported type
@@ -29,6 +39,23 @@ public UnsupportedEsField(String name, String originalType, String inherited, Ma
         this.inherited = inherited;
     }
 
+    public UnsupportedEsField(StreamInput in) throws IOException {
+        this(in.readString(), in.readString(), in.readOptionalString(), in.readMap(i -> i.readNamedWriteable(EsField.class)));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(getName());
+        out.writeString(getOriginalType());
+        out.writeOptionalString(getInherited());
+        out.writeMap(getProperties(), StreamOutput::writeNamedWriteable);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     public String getOriginalType() {
         return originalType;
     }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java
new file mode 100644
index 0000000000000..372bd7ec8066a
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.type;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.AbstractNamedWriteableTestCase;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+public abstract class AbstractEsFieldTypeTests<T extends EsField> extends AbstractNamedWriteableTestCase<EsField> {
+    @Override
+    protected abstract T createTestInstance();
+
+    protected abstract T mutate(T instance) throws IOException;
+
+    /**
+     * Generate sub-properties.
+     * @param maxDepth the maximum number of levels of properties to make
+     */
+    static Map<String, EsField> randomProperties(int maxDepth) {
+        if (maxDepth < 0) {
+            throw new IllegalArgumentException("depth must be >= 0");
+        }
+        if (maxDepth == 0 || randomBoolean()) {
+            return Map.of();
+        }
+        int targetSize = between(1, 5);
+        Map<String, EsField> properties = new TreeMap<>();
+        while (properties.size() < targetSize) {
+            properties.put(randomAlphaOfLength(properties.size() + 1), switch (between(0, 5)) {
+                case 0 -> EsFieldTests.randomEsField(maxDepth - 1);
+                case 1 -> DateEsFieldTests.randomDateEsField(maxDepth - 1);
+                case 2 -> InvalidMappedFieldTests.randomInvalidMappedField(maxDepth - 1);
+                case 3 -> KeywordEsFieldTests.randomKeywordEsField(maxDepth - 1);
+                case 4 -> TextEsFieldTests.randomTextEsField(maxDepth - 1);
+                case 5 -> UnsupportedEsFieldTests.randomUnsupportedEsField(maxDepth - 1);
+                default -> throw new IllegalArgumentException();
+            });
+        }
+        return properties;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected final T mutateInstance(EsField instance) throws IOException {
+        return mutate((T) instance);
+    }
+
+    @Override
+    protected final NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(EsField.getNamedWriteables());
+    }
+
+    @Override
+    protected final Class<EsField> categoryClass() {
+        return EsField.class;
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java
new file mode 100644
index 0000000000000..c6428034eaae1
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.type;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class DateEsFieldTests extends AbstractEsFieldTypeTests<DateEsField> {
+    static DateEsField randomDateEsField(int maxPropertiesDepth) {
+        return DateEsField.dateEsField(randomAlphaOfLength(5), randomProperties(maxPropertiesDepth), randomBoolean());
+    }
+
+    @Override
+    protected DateEsField createTestInstance() {
+        return randomDateEsField(4);
+    }
+
+    @Override
+    protected DateEsField mutate(DateEsField instance) throws IOException {
+        String name = instance.getName();
+        Map<String, EsField> properties = instance.getProperties();
+        boolean aggregatable = instance.isAggregatable();
+        switch (between(0, 2)) {
+            case 0 -> name = randomAlphaOfLength(name.length() + 1);
+            case 1 -> properties = randomValueOtherThan(properties, () -> randomProperties(4));
+            case 2 -> aggregatable = false == aggregatable;
+            default -> throw new IllegalArgumentException();
+        }
+        return DateEsField.dateEsField(name, properties, aggregatable);
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
new file mode 100644
index 0000000000000..2d75def3296d3
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.type;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class EsFieldTests extends AbstractEsFieldTypeTests<EsField> {
+    static EsField randomEsField(int maxPropertiesDepth) {
+        String name = randomAlphaOfLength(4);
+        DataType esDataType = randomFrom(DataTypes.types());
+        Map<String, EsField> properties = randomProperties(maxPropertiesDepth);
+        boolean aggregatable = randomBoolean();
+        boolean isAlias = randomBoolean();
+        return new EsField(name, esDataType, properties, aggregatable, isAlias);
+    }
+
+    @Override
+    protected EsField createTestInstance() {
+        return randomEsField(4);
+    }
+
+    @Override
+    protected EsField mutate(EsField instance) throws IOException {
+        String name = instance.getName();
+        DataType esDataType = instance.getDataType();
+        Map<String, EsField> properties = instance.getProperties();
+        boolean aggregatable = instance.isAggregatable();
+        boolean isAlias = instance.isAlias();
+        switch (between(0, 4)) {
+            case 0 -> name = randomAlphaOfLength(name.length() + 1);
+            case 1 -> esDataType = randomValueOtherThan(esDataType, () -> randomFrom(DataTypes.types()));
+            case 2 -> properties = randomValueOtherThan(properties, () -> randomProperties(4));
+            case 3 -> aggregatable = false == aggregatable;
+            case 4 -> isAlias = false == isAlias;
+            default -> throw new IllegalArgumentException();
+        }
+        return new EsField(name, esDataType, properties, aggregatable, isAlias);
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java
new file mode 100644
index 0000000000000..5f96b4a720381
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.type;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class InvalidMappedFieldTests extends AbstractEsFieldTypeTests<InvalidMappedField> {
+    static InvalidMappedField randomInvalidMappedField(int maxPropertiesDepth) {
+        String name = randomAlphaOfLength(4);
+        String errorMessage = randomAlphaOfLengthBetween(1, 100);
+        Map<String, EsField> properties = randomProperties(maxPropertiesDepth);
+        return new InvalidMappedField(name, errorMessage, properties);
+    }
+
+    @Override
+    protected InvalidMappedField createTestInstance() {
+        return randomInvalidMappedField(4);
+    }
+
+    @Override
+    protected InvalidMappedField mutate(InvalidMappedField instance) throws IOException {
+        String name = instance.getName();
+        String errorMessage = instance.errorMessage();
+        Map<String, EsField> properties = instance.getProperties();
+        switch (between(0, 2)) {
+            case 0 -> name = randomAlphaOfLength(name.length() + 1);
+            case 1 -> errorMessage = randomValueOtherThan(errorMessage, () -> randomAlphaOfLengthBetween(1, 100));
+            case 2 -> properties = randomValueOtherThan(properties, () -> randomProperties(4));
+            default -> throw new IllegalArgumentException();
+        }
+        return new InvalidMappedField(name, errorMessage, properties);
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java
new file mode 100644
index 0000000000000..0b1c866114737
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.type;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class KeywordEsFieldTests extends AbstractEsFieldTypeTests<KeywordEsField> {
+    static KeywordEsField randomKeywordEsField(int maxPropertiesDepth) {
+        String name = randomAlphaOfLength(4);
+        Map<String, EsField> properties = randomProperties(maxPropertiesDepth);
+        boolean hasDocValues = randomBoolean();
+        int precision = randomInt();
+        boolean normalized = randomBoolean();
+        boolean isAlias = randomBoolean();
+        return new KeywordEsField(name, properties, hasDocValues, precision, normalized, isAlias);
+    }
+
+    @Override
+    protected KeywordEsField createTestInstance() {
+        return randomKeywordEsField(4);
+    }
+
+    @Override
+    protected KeywordEsField mutate(KeywordEsField instance) throws IOException {
+        String name = instance.getName();
+        Map<String, EsField> properties = instance.getProperties();
+        boolean hasDocValues = instance.isAggregatable();
+        int precision = instance.getPrecision();
+        boolean normalized = instance.getNormalized();
+        boolean isAlias = instance.isAlias();
+        switch (between(0, 5)) {
+            case 0 -> name = randomAlphaOfLength(name.length() + 1);
+            case 1 -> properties = randomValueOtherThan(properties, () -> randomProperties(4));
+            case 2 -> hasDocValues = false == hasDocValues;
+            case 3 -> precision = randomValueOtherThan(precision, ESTestCase::randomInt);
+            case 4 -> normalized = false == normalized;
+            case 5 -> isAlias = false == isAlias;
+            default -> throw new IllegalArgumentException();
+        }
+        return new KeywordEsField(name, properties, hasDocValues, precision, normalized, isAlias);
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java
new file mode 100644
index 0000000000000..df00bf27c32b9
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.type;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class TextEsFieldTests extends AbstractEsFieldTypeTests<TextEsField> {
+    static TextEsField randomTextEsField(int maxPropertiesDepth) {
+        String name = randomAlphaOfLength(4);
+        Map<String, EsField> properties = randomProperties(maxPropertiesDepth);
+        boolean hasDocValues = randomBoolean();
+        boolean isAlias = randomBoolean();
+        return new TextEsField(name, properties, hasDocValues, isAlias);
+    }
+
+    @Override
+    protected TextEsField createTestInstance() {
+        return randomTextEsField(4);
+    }
+
+    @Override
+    protected TextEsField mutate(TextEsField instance) throws IOException {
+        String name = instance.getName();
+        Map<String, EsField> properties = instance.getProperties();
+        boolean hasDocValues = instance.isAggregatable();
+        boolean isAlias = instance.isAlias();
+        switch (between(0, 3)) {
+            case 0 -> name = randomAlphaOfLength(name.length() + 1);
+            case 1 -> properties = randomValueOtherThan(properties, () -> randomProperties(4));
+            case 2 -> hasDocValues = false == hasDocValues;
+            case 3 -> isAlias = false == isAlias;
+            default -> throw new IllegalArgumentException();
+        }
+        return new TextEsField(name, properties, hasDocValues, isAlias);
+    }
+}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java
new file mode 100644
index 0000000000000..fb1ca014a0ee0
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.type;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class UnsupportedEsFieldTests extends AbstractEsFieldTypeTests<UnsupportedEsField> {
+    static UnsupportedEsField randomUnsupportedEsField(int maxPropertiesDepth) {
+        String name = randomAlphaOfLength(4);
+        String originalType = randomAlphaOfLength(5);
+        String inherited = randomBoolean() ? null : randomAlphaOfLength(5);
+        Map<String, EsField> properties = randomProperties(maxPropertiesDepth);
+        return new UnsupportedEsField(name, originalType, inherited, properties);
+    }
+
+    @Override
+    protected UnsupportedEsField createTestInstance() {
+        return randomUnsupportedEsField(4);
+    }
+
+    @Override
+    protected UnsupportedEsField mutate(UnsupportedEsField instance) throws IOException {
+        String name = instance.getName();
+        String originalType = randomAlphaOfLength(5);
+        String inherited = randomBoolean() ? null : randomAlphaOfLength(5);
+        Map<String, EsField> properties = instance.getProperties();
+        switch (between(0, 3)) {
+            case 0 -> name = randomAlphaOfLength(name.length() + 1);
+            case 1 -> originalType = randomValueOtherThan(originalType, () -> randomAlphaOfLength(4));
+            case 2 -> inherited = randomValueOtherThan(inherited, () -> randomBoolean() ? null : randomAlphaOfLength(4));
+            case 3 -> properties = randomValueOtherThan(properties, () -> randomProperties(4));
+            default -> throw new IllegalArgumentException();
+        }
+        return new UnsupportedEsField(name, originalType, inherited, properties);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index 7a7cbb55f5629..c0859782937bc 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -50,11 +50,7 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DateEsField;
 import org.elasticsearch.xpack.esql.core.type.EsField;
-import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
-import org.elasticsearch.xpack.esql.core.type.KeywordEsField;
-import org.elasticsearch.xpack.esql.core.type.TextEsField;
 import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
@@ -307,13 +303,6 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(Attribute.class, ReferenceAttribute.class, PlanNamedTypes::writeReferenceAttr, PlanNamedTypes::readReferenceAttr),
             of(Attribute.class, MetadataAttribute.class, PlanNamedTypes::writeMetadataAttr, PlanNamedTypes::readMetadataAttr),
             of(Attribute.class, UnsupportedAttribute.class, PlanNamedTypes::writeUnsupportedAttr, PlanNamedTypes::readUnsupportedAttr),
-            // EsFields
-            of(EsField.class, EsField.class, PlanNamedTypes::writeEsField, PlanNamedTypes::readEsField),
-            of(EsField.class, DateEsField.class, PlanNamedTypes::writeDateEsField, PlanNamedTypes::readDateEsField),
-            of(EsField.class, InvalidMappedField.class, PlanNamedTypes::writeInvalidMappedField, PlanNamedTypes::readInvalidMappedField),
-            of(EsField.class, KeywordEsField.class, PlanNamedTypes::writeKeywordEsField, PlanNamedTypes::readKeywordEsField),
-            of(EsField.class, TextEsField.class, PlanNamedTypes::writeTextEsField, PlanNamedTypes::readTextEsField),
-            of(EsField.class, UnsupportedEsField.class, PlanNamedTypes::writeUnsupportedEsField, PlanNamedTypes::readUnsupportedEsField),
             // NamedExpressions
             of(NamedExpression.class, Alias.class, PlanNamedTypes::writeAlias, PlanNamedTypes::readAlias),
             // BinaryComparison
@@ -1071,7 +1060,7 @@ static FieldAttribute readFieldAttribute(PlanStreamInput in) throws IOException
             in.readOptionalWithReader(PlanNamedTypes::readFieldAttribute),
             in.readString(),
             DataType.readFrom(in),
-            in.readEsFieldNamed(),
+            in.readNamedWriteable(EsField.class),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
             NameId.readFrom(in),
@@ -1084,7 +1073,7 @@ static void writeFieldAttribute(PlanStreamOutput out, FieldAttribute fieldAttrib
         out.writeOptionalWriteable(fieldAttribute.parent() == null ? null : o -> writeFieldAttribute(out, fieldAttribute.parent()));
         out.writeString(fieldAttribute.name());
         out.writeString(fieldAttribute.dataType().typeName());
-        out.writeNamed(EsField.class, fieldAttribute.field());
+        out.writeNamedWriteable(fieldAttribute.field());
         out.writeOptionalString(fieldAttribute.qualifier());
         out.writeEnum(fieldAttribute.nullable());
         fieldAttribute.id().writeTo(out);
@@ -1141,7 +1130,7 @@ static UnsupportedAttribute readUnsupportedAttr(PlanStreamInput in) throws IOExc
         return new UnsupportedAttribute(
             Source.readFrom(in),
             in.readString(),
-            readUnsupportedEsField(in),
+            new UnsupportedEsField(in),
             in.readOptionalString(),
             NameId.readFrom(in)
         );
@@ -1150,111 +1139,11 @@ static UnsupportedAttribute readUnsupportedAttr(PlanStreamInput in) throws IOExc
     static void writeUnsupportedAttr(PlanStreamOutput out, UnsupportedAttribute unsupportedAttribute) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writeString(unsupportedAttribute.name());
-        writeUnsupportedEsField(out, unsupportedAttribute.field());
+        unsupportedAttribute.field().writeTo(out);
         out.writeOptionalString(unsupportedAttribute.hasCustomMessage() ? unsupportedAttribute.unresolvedMessage() : null);
         unsupportedAttribute.id().writeTo(out);
     }
 
-    // -- EsFields
-
-    static EsField readEsField(PlanStreamInput in) throws IOException {
-        return new EsField(
-            in.readString(),
-            DataType.readFrom(in),
-            in.readImmutableMap(StreamInput::readString, readerFromPlanReader(PlanStreamInput::readEsFieldNamed)),
-            in.readBoolean(),
-            in.readBoolean()
-        );
-    }
-
-    static void writeEsField(PlanStreamOutput out, EsField esField) throws IOException {
-        out.writeString(esField.getName());
-        out.writeString(esField.getDataType().typeName());
-        out.writeMap(esField.getProperties(), (o, v) -> out.writeNamed(EsField.class, v));
-        out.writeBoolean(esField.isAggregatable());
-        out.writeBoolean(esField.isAlias());
-    }
-
-    static DateEsField readDateEsField(PlanStreamInput in) throws IOException {
-        return DateEsField.dateEsField(
-            in.readString(),
-            in.readImmutableMap(StreamInput::readString, readerFromPlanReader(PlanStreamInput::readEsFieldNamed)),
-            in.readBoolean()
-        );
-    }
-
-    static void writeDateEsField(PlanStreamOutput out, DateEsField dateEsField) throws IOException {
-        out.writeString(dateEsField.getName());
-        out.writeMap(dateEsField.getProperties(), (o, v) -> out.writeNamed(EsField.class, v));
-        out.writeBoolean(dateEsField.isAggregatable());
-    }
-
-    static InvalidMappedField readInvalidMappedField(PlanStreamInput in) throws IOException {
-        return new InvalidMappedField(
-            in.readString(),
-            in.readString(),
-            in.readImmutableMap(StreamInput::readString, readerFromPlanReader(PlanStreamInput::readEsFieldNamed))
-        );
-    }
-
-    static void writeInvalidMappedField(PlanStreamOutput out, InvalidMappedField field) throws IOException {
-        out.writeString(field.getName());
-        out.writeString(field.errorMessage());
-        out.writeMap(field.getProperties(), (o, v) -> out.writeNamed(EsField.class, v));
-    }
-
-    static KeywordEsField readKeywordEsField(PlanStreamInput in) throws IOException {
-        return new KeywordEsField(
-            in.readString(),
-            in.readImmutableMap(StreamInput::readString, readerFromPlanReader(PlanStreamInput::readEsFieldNamed)),
-            in.readBoolean(),
-            in.readInt(),
-            in.readBoolean(),
-            in.readBoolean()
-        );
-    }
-
-    static void writeKeywordEsField(PlanStreamOutput out, KeywordEsField keywordEsField) throws IOException {
-        out.writeString(keywordEsField.getName());
-        out.writeMap(keywordEsField.getProperties(), (o, v) -> out.writeNamed(EsField.class, v));
-        out.writeBoolean(keywordEsField.isAggregatable());
-        out.writeInt(keywordEsField.getPrecision());
-        out.writeBoolean(keywordEsField.getNormalized());
-        out.writeBoolean(keywordEsField.isAlias());
-    }
-
-    static TextEsField readTextEsField(PlanStreamInput in) throws IOException {
-        return new TextEsField(
-            in.readString(),
-            in.readImmutableMap(StreamInput::readString, readerFromPlanReader(PlanStreamInput::readEsFieldNamed)),
-            in.readBoolean(),
-            in.readBoolean()
-        );
-    }
-
-    static void writeTextEsField(PlanStreamOutput out, TextEsField textEsField) throws IOException {
-        out.writeString(textEsField.getName());
-        out.writeMap(textEsField.getProperties(), (o, v) -> out.writeNamed(EsField.class, v));
-        out.writeBoolean(textEsField.isAggregatable());
-        out.writeBoolean(textEsField.isAlias());
-    }
-
-    static UnsupportedEsField readUnsupportedEsField(PlanStreamInput in) throws IOException {
-        return new UnsupportedEsField(
-            in.readString(),
-            in.readString(),
-            in.readOptionalString(),
-            in.readImmutableMap(StreamInput::readString, readerFromPlanReader(PlanStreamInput::readEsFieldNamed))
-        );
-    }
-
-    static void writeUnsupportedEsField(PlanStreamOutput out, UnsupportedEsField unsupportedEsField) throws IOException {
-        out.writeString(unsupportedEsField.getName());
-        out.writeString(unsupportedEsField.getOriginalType());
-        out.writeOptionalString(unsupportedEsField.getInherited());
-        out.writeMap(unsupportedEsField.getProperties(), (o, v) -> out.writeNamed(EsField.class, v));
-    }
-
     // -- BinaryComparison
 
     static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) throws IOException {
@@ -1967,14 +1856,14 @@ static void writeFieldSort(PlanStreamOutput out, EsQueryExec.FieldSort fieldSort
     static EsIndex readEsIndex(PlanStreamInput in) throws IOException {
         return new EsIndex(
             in.readString(),
-            in.readImmutableMap(StreamInput::readString, readerFromPlanReader(PlanStreamInput::readEsFieldNamed)),
+            in.readImmutableMap(StreamInput::readString, i -> i.readNamedWriteable(EsField.class)),
             (Set<String>) in.readGenericValue()
         );
     }
 
     static void writeEsIndex(PlanStreamOutput out, EsIndex esIndex) throws IOException {
         out.writeString(esIndex.name());
-        out.writeMap(esIndex.mapping(), (o, v) -> out.writeNamed(EsField.class, v));
+        out.writeMap(esIndex.mapping(), StreamOutput::writeNamedWriteable);
         out.writeGenericValue(esIndex.concreteIndices());
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
index 6c1b7db24fe46..73a7ba6549dc8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanNamedReader;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
@@ -111,10 +110,6 @@ public Attribute readAttribute() throws IOException {
         return readNamed(Attribute.class);
     }
 
-    public EsField readEsFieldNamed() throws IOException {
-        return readNamed(EsField.class);
-    }
-
     public <T> T readNamed(Class<T> type) throws IOException {
         String name = readString();
         @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
index 833e540e37cf6..f3b06f5629524 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
@@ -56,6 +56,7 @@
 import org.elasticsearch.xpack.esql.action.RestEsqlGetAsyncResultAction;
 import org.elasticsearch.xpack.esql.action.RestEsqlQueryAction;
 import org.elasticsearch.xpack.esql.core.index.IndexResolver;
+import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator;
 import org.elasticsearch.xpack.esql.execution.PlanExecutor;
 import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery;
@@ -63,13 +64,13 @@
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
 
 import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
-import java.util.stream.Stream;
 
 public class EsqlPlugin extends Plugin implements ActionPlugin {
 
@@ -172,26 +173,25 @@ public List<RestHandler> getRestHandlers(
 
     @Override
     public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
-        return Stream.concat(
-            List.of(
-                DriverStatus.ENTRY,
-                AbstractPageMappingOperator.Status.ENTRY,
-                AbstractPageMappingToIteratorOperator.Status.ENTRY,
-                AggregationOperator.Status.ENTRY,
-                ExchangeSinkOperator.Status.ENTRY,
-                ExchangeSourceOperator.Status.ENTRY,
-                HashAggregationOperator.Status.ENTRY,
-                LimitOperator.Status.ENTRY,
-                LuceneOperator.Status.ENTRY,
-                TopNOperatorStatus.ENTRY,
-                MvExpandOperator.Status.ENTRY,
-                ValuesSourceReaderOperator.Status.ENTRY,
-                SingleValueQuery.ENTRY,
-                AsyncOperator.Status.ENTRY,
-                EnrichLookupOperator.Status.ENTRY
-            ).stream(),
-            Block.getNamedWriteables().stream()
-        ).toList();
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.add(DriverStatus.ENTRY);
+        entries.add(AbstractPageMappingOperator.Status.ENTRY);
+        entries.add(AbstractPageMappingToIteratorOperator.Status.ENTRY);
+        entries.add(AggregationOperator.Status.ENTRY);
+        entries.add(ExchangeSinkOperator.Status.ENTRY);
+        entries.add(ExchangeSourceOperator.Status.ENTRY);
+        entries.add(HashAggregationOperator.Status.ENTRY);
+        entries.add(LimitOperator.Status.ENTRY);
+        entries.add(LuceneOperator.Status.ENTRY);
+        entries.add(TopNOperatorStatus.ENTRY);
+        entries.add(MvExpandOperator.Status.ENTRY);
+        entries.add(ValuesSourceReaderOperator.Status.ENTRY);
+        entries.add(SingleValueQuery.ENTRY);
+        entries.add(AsyncOperator.Status.ENTRY);
+        entries.add(EnrichLookupOperator.Status.ENTRY);
+        entries.addAll(Block.getNamedWriteables());
+        entries.addAll(EsField.getNamedWriteables());
+        return entries;
     }
 
     public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
index 047b4902ac59d..3828b1b290c8f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
@@ -34,6 +35,7 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.ArrayList;
 import java.util.List;
 
 public class SerializationTestUtils {
@@ -101,18 +103,17 @@ public interface Deserializer<T> {
     }
 
     public static NamedWriteableRegistry writableRegistry() {
-        return new NamedWriteableRegistry(
-            List.of(
-                new NamedWriteableRegistry.Entry(QueryBuilder.class, TermQueryBuilder.NAME, TermQueryBuilder::new),
-                new NamedWriteableRegistry.Entry(QueryBuilder.class, TermsQueryBuilder.NAME, TermsQueryBuilder::new),
-                new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new),
-                new NamedWriteableRegistry.Entry(QueryBuilder.class, RangeQueryBuilder.NAME, RangeQueryBuilder::new),
-                new NamedWriteableRegistry.Entry(QueryBuilder.class, BoolQueryBuilder.NAME, BoolQueryBuilder::new),
-                new NamedWriteableRegistry.Entry(QueryBuilder.class, WildcardQueryBuilder.NAME, WildcardQueryBuilder::new),
-                new NamedWriteableRegistry.Entry(QueryBuilder.class, RegexpQueryBuilder.NAME, RegexpQueryBuilder::new),
-                new NamedWriteableRegistry.Entry(QueryBuilder.class, ExistsQueryBuilder.NAME, ExistsQueryBuilder::new),
-                SingleValueQuery.ENTRY
-            )
-        );
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, TermQueryBuilder.NAME, TermQueryBuilder::new));
+        entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, TermsQueryBuilder.NAME, TermsQueryBuilder::new));
+        entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
+        entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, RangeQueryBuilder.NAME, RangeQueryBuilder::new));
+        entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, BoolQueryBuilder.NAME, BoolQueryBuilder::new));
+        entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, WildcardQueryBuilder.NAME, WildcardQueryBuilder::new));
+        entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, RegexpQueryBuilder.NAME, RegexpQueryBuilder::new));
+        entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, ExistsQueryBuilder.NAME, ExistsQueryBuilder::new));
+        entries.add(SingleValueQuery.ENTRY);
+        entries.addAll(EsField.getNamedWriteables());
+        return new NamedWriteableRegistry(entries);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
index e05abefba66a7..c1a7f1219c5f7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
-import org.elasticsearch.xpack.esql.core.type.DateEsField;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
 import org.elasticsearch.xpack.esql.core.type.KeywordEsField;
@@ -266,66 +265,6 @@ public void testFieldAttribute() {
         Stream.generate(PlanNamedTypesTests::randomFieldAttribute).limit(100).forEach(PlanNamedTypesTests::assertNamedExpression);
     }
 
-    public void testKeywordEsFieldSimple() throws IOException {
-        var orig = new KeywordEsField(
-            "BarKeyField", // name
-            Map.of(), // no properties
-            true, // hasDocValues
-            5, // precision
-            true, // normalized
-            true // alias
-        );
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeKeywordEsField(out, orig);
-        var deser = PlanNamedTypes.readKeywordEsField(planStreamInput(bso));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-    }
-
-    public void testKeywordEsField() {
-        Stream.generate(PlanNamedTypesTests::randomKeywordEsField).limit(100).forEach(PlanNamedTypesTests::assertNamedEsField);
-    }
-
-    public void testTextdEsFieldSimple() throws IOException {
-        var orig = new TextEsField(
-            "BarKeyField", // name
-            Map.of(), // no properties
-            true, // hasDocValues
-            true // alias
-        );
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeTextEsField(out, orig);
-        var deser = PlanNamedTypes.readTextEsField(planStreamInput(bso));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-    }
-
-    public void testTextEsField() {
-        Stream.generate(PlanNamedTypesTests::randomTextEsField).limit(100).forEach(PlanNamedTypesTests::assertNamedEsField);
-    }
-
-    public void testInvalidMappedFieldSimple() throws IOException {
-        var orig = new InvalidMappedField("foo", "bar");
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeInvalidMappedField(out, orig);
-        var deser = PlanNamedTypes.readInvalidMappedField(planStreamInput(bso));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-    }
-
-    public void testInvalidMappedField() {
-        Stream.generate(PlanNamedTypesTests::randomInvalidMappedField).limit(100).forEach(PlanNamedTypesTests::assertNamedEsField);
-    }
-
-    public void testEsDateFieldSimple() throws IOException {
-        var orig = DateEsField.dateEsField("birth_date", Map.of(), false);
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeDateEsField(out, orig);
-        var deser = PlanNamedTypes.readDateEsField(planStreamInput(bso));
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-    }
-
     public void testBinComparisonSimple() throws IOException {
         var orig = new Equals(Source.EMPTY, field("foo", DataTypes.DOUBLE), field("bar", DataTypes.DOUBLE));
         BytesStreamOutput bso = new BytesStreamOutput();
@@ -516,11 +455,6 @@ private static <T> void assertNamedType(Class<T> type, T origObj) {
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(origObj, unused -> deserObj);
     }
 
-    private static void assertNamedEsField(EsField origObj) {
-        var deserObj = serializeDeserialize(origObj, (o, v) -> o.writeNamed(EsField.class, v), PlanStreamInput::readEsFieldNamed);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(origObj, unused -> deserObj);
-    }
-
     static EsIndex randomEsIndex() {
         return new EsIndex(
             randomAlphaOfLength(randomIntBetween(1, 25)),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
index d45141ab4c477..2b2a4e76c5cc8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
@@ -59,6 +59,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() {
         List<NamedWriteableRegistry.Entry> writeables = new ArrayList<>();
         writeables.addAll(new SearchModule(Settings.EMPTY, List.of()).getNamedWriteables());
         writeables.addAll(Block.getNamedWriteables());
+        writeables.addAll(EsField.getNamedWriteables());
         return new NamedWriteableRegistry(writeables);
     }
 

From 6532af82ce3cf80a6d216ea27852d68f6579900e Mon Sep 17 00:00:00 2001
From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com>
Date: Fri, 31 May 2024 10:14:28 -0400
Subject: [PATCH 082/208] [Inference API] Check pipelines on delete inference
 endpoint (#109123)

* renaming

* check for pipeline references

* add force and dry run options to request

* add force delete and dry run options to Actions

* Revert dependency on ml module

* duplicate InferenceProcessorInfoExtractor to inference module

* Ignore missing IngestMetadata during attempted endpoint delete

* Update docs/changelog/109123.yaml

* rename RestDeleteInference Action

* fix dry run

* Add integration test and misc improvements

* Update 109123.yaml

* merge fixes

* Improvements from PR review

* Moving shared pieces to core (#109183)

* replace `model` in exception message with `inference endpoint`

* test fix

---------

Co-authored-by: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com>
---
 docs/changelog/109123.yaml                    |   5 +
 .../org/elasticsearch/TransportVersions.java  |   1 +
 .../action/DeleteInferenceEndpointAction.java | 142 ++++++++++++++
 .../action/DeleteInferenceModelAction.java    |  74 --------
 .../ml/utils/InferenceProcessorConstants.java |  20 ++
 .../InferenceProcessorInfoExtractor.java      |  39 +++-
 .../InferenceProcessorInfoExtractorTests.java |  11 +-
 .../inference/InferenceBaseRestTest.java      |  30 +++
 .../xpack/inference/InferenceCrudIT.java      |  31 ++++
 .../xpack/inference/InferencePlugin.java      |  10 +-
 ...ransportDeleteInferenceEndpointAction.java | 173 ++++++++++++++++++
 .../TransportDeleteInferenceModelAction.java  | 104 -----------
 ...=> RestDeleteInferenceEndpointAction.java} |  18 +-
 .../ml/integration/TestFeatureResetIT.java    |   2 +-
 .../xpack/ml/MachineLearning.java             |   2 +-
 .../TransportDeleteTrainedModelAction.java    |   2 +-
 ...ransportDeleteTrainedModelAliasAction.java |   2 +-
 .../TransportGetTrainedModelsStatsAction.java |   2 +-
 ...sportStopTrainedModelDeploymentAction.java |   2 +-
 .../inference/ingest/InferenceProcessor.java  |  11 +-
 20 files changed, 473 insertions(+), 208 deletions(-)
 create mode 100644 docs/changelog/109123.yaml
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceModelAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorConstants.java
 rename x-pack/plugin/{ml/src/main/java/org/elasticsearch/xpack => core/src/main/java/org/elasticsearch/xpack/core}/ml/utils/InferenceProcessorInfoExtractor.java (82%)
 rename x-pack/plugin/{ml/src/test/java/org/elasticsearch/xpack => core/src/test/java/org/elasticsearch/xpack/core}/ml/utils/InferenceProcessorInfoExtractorTests.java (96%)
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
 delete mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java
 rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/{RestDeleteInferenceModelAction.java => RestDeleteInferenceEndpointAction.java} (75%)

diff --git a/docs/changelog/109123.yaml b/docs/changelog/109123.yaml
new file mode 100644
index 0000000000000..dfd7e52b33e7f
--- /dev/null
+++ b/docs/changelog/109123.yaml
@@ -0,0 +1,5 @@
+pr: 109123
+summary: "[Inference API] Check for related pipelines on delete inference endpoint"
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index f10c079c2e82c..e5428497c136d 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -180,6 +180,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ADD_METADATA_FLATTENED_TO_ROLES = def(8_671_00_0);
     public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED = def(8_672_00_0);
     public static final TransportVersion WATCHER_REQUEST_TIMEOUTS = def(8_673_00_0);
+    public static final TransportVersion ML_INFERENCE_ENHANCE_DELETE_ENDPOINT = def(8_674_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java
new file mode 100644
index 0000000000000..be4c87195e9d1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.inference.action;
+
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
+public class DeleteInferenceEndpointAction extends ActionType<AcknowledgedResponse> {
+
+    public static final DeleteInferenceEndpointAction INSTANCE = new DeleteInferenceEndpointAction();
+    public static final String NAME = "cluster:admin/xpack/inference/delete";
+
+    public DeleteInferenceEndpointAction() {
+        super(NAME);
+    }
+
+    public static class Request extends AcknowledgedRequest<DeleteInferenceEndpointAction.Request> {
+
+        private final String inferenceEndpointId;
+        private final TaskType taskType;
+        private final boolean forceDelete;
+        private final boolean dryRun;
+
+        public Request(String inferenceEndpointId, TaskType taskType, boolean forceDelete, boolean dryRun) {
+            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+            this.inferenceEndpointId = inferenceEndpointId;
+            this.taskType = taskType;
+            this.forceDelete = forceDelete;
+            this.dryRun = dryRun;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.inferenceEndpointId = in.readString();
+            this.taskType = TaskType.fromStream(in);
+            if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ENHANCE_DELETE_ENDPOINT)) {
+                this.forceDelete = Boolean.TRUE.equals(in.readOptionalBoolean());
+                this.dryRun = Boolean.TRUE.equals(in.readOptionalBoolean());
+            } else {
+                this.forceDelete = false;
+                this.dryRun = false;
+            }
+        }
+
+        public String getInferenceEndpointId() {
+            return inferenceEndpointId;
+        }
+
+        public TaskType getTaskType() {
+            return taskType;
+        }
+
+        public boolean isForceDelete() {
+            return forceDelete;
+        }
+
+        public boolean isDryRun() {
+            return dryRun;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(inferenceEndpointId);
+            taskType.writeTo(out);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_ENHANCE_DELETE_ENDPOINT)) {
+                out.writeOptionalBoolean(forceDelete);
+                out.writeOptionalBoolean(dryRun);
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            DeleteInferenceEndpointAction.Request request = (DeleteInferenceEndpointAction.Request) o;
+            return Objects.equals(inferenceEndpointId, request.inferenceEndpointId)
+                && taskType == request.taskType
+                && forceDelete == request.forceDelete
+                && dryRun == request.dryRun;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(inferenceEndpointId, taskType, forceDelete, dryRun);
+        }
+    }
+
+    public static class Response extends AcknowledgedResponse {
+
+        private final String PIPELINE_IDS = "pipelines";
+        Set<String> pipelineIds;
+
+        public Response(boolean acknowledged, Set<String> pipelineIds) {
+            super(acknowledged);
+            this.pipelineIds = pipelineIds;
+        }
+
+        public Response(StreamInput in) throws IOException {
+            super(in);
+            pipelineIds = in.readCollectionAsSet(StreamInput::readString);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeCollection(pipelineIds, StreamOutput::writeString);
+        }
+
+        @Override
+        protected void addCustomFields(XContentBuilder builder, Params params) throws IOException {
+            super.addCustomFields(builder, params);
+            builder.field(PIPELINE_IDS, pipelineIds);
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder returnable = new StringBuilder();
+            returnable.append("acknowledged: ").append(this.acknowledged);
+            returnable.append(", pipelineIdsByEndpoint: ");
+            for (String entry : pipelineIds) {
+                returnable.append(entry).append(", ");
+            }
+            return returnable.toString();
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceModelAction.java
deleted file mode 100644
index 03c41e54ad2bb..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceModelAction.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.inference.action;
-
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.support.master.AcknowledgedRequest;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.inference.TaskType;
-
-import java.io.IOException;
-import java.util.Objects;
-
-public class DeleteInferenceModelAction extends ActionType<AcknowledgedResponse> {
-
-    public static final DeleteInferenceModelAction INSTANCE = new DeleteInferenceModelAction();
-    public static final String NAME = "cluster:admin/xpack/inference/delete";
-
-    public DeleteInferenceModelAction() {
-        super(NAME);
-    }
-
-    public static class Request extends AcknowledgedRequest<DeleteInferenceModelAction.Request> {
-
-        private final String inferenceEntityId;
-        private final TaskType taskType;
-
-        public Request(String inferenceEntityId, TaskType taskType) {
-            super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
-            this.inferenceEntityId = inferenceEntityId;
-            this.taskType = taskType;
-        }
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            this.inferenceEntityId = in.readString();
-            this.taskType = TaskType.fromStream(in);
-        }
-
-        public String getInferenceEntityId() {
-            return inferenceEntityId;
-        }
-
-        public TaskType getTaskType() {
-            return taskType;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeString(inferenceEntityId);
-            taskType.writeTo(out);
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            DeleteInferenceModelAction.Request request = (DeleteInferenceModelAction.Request) o;
-            return Objects.equals(inferenceEntityId, request.inferenceEntityId) && taskType == request.taskType;
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(inferenceEntityId, taskType);
-        }
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorConstants.java
new file mode 100644
index 0000000000000..7222f20784bb7
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorConstants.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml.utils;
+
+/**
+ * Provides access to constants for core, ml, and inference plugins
+ */
+public class InferenceProcessorConstants {
+    public static final String TYPE = "inference";
+    public static final String TARGET_FIELD = "target_field";
+    public static final String FIELD_MAP = "field_map";
+    public static final String INFERENCE_CONFIG = "inference_config";
+
+    private InferenceProcessorConstants() {}
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/InferenceProcessorInfoExtractor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractor.java
similarity index 82%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/InferenceProcessorInfoExtractor.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractor.java
index fe16cc4c59bed..1be495a8a82f5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/InferenceProcessorInfoExtractor.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractor.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.ml.utils;
+package org.elasticsearch.xpack.core.ml.utils;
 
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.cluster.ClusterState;
@@ -16,6 +16,7 @@
 import org.elasticsearch.transport.Transports;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -24,13 +25,11 @@
 
 import static org.elasticsearch.inference.InferenceResults.MODEL_ID_RESULTS_FIELD;
 import static org.elasticsearch.ingest.Pipeline.PROCESSORS_KEY;
-import static org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor.TYPE;
 
 /**
  * Utilities for extracting information around inference processors from IngestMetadata
  */
 public final class InferenceProcessorInfoExtractor {
-
     private static final String FOREACH_PROCESSOR_NAME = "foreach";
     // Any more than 10 nestings of processors, we stop searching for inference processor definitions
     private static final int MAX_INFERENCE_PROCESSOR_SEARCH_RECURSIONS = 10;
@@ -131,6 +130,38 @@ public static Map<String, Set<String>> pipelineIdsByResource(ClusterState state,
         return pipelineIdsByModelIds;
     }
 
+    /**
+     * @param state Current {@link ClusterState}
+     * @return a map from Model or Deployment IDs or Aliases to each pipeline referencing them.
+     */
+    @SuppressWarnings("unchecked")
+    public static Set<String> pipelineIdsForResource(ClusterState state, Set<String> ids) {
+        assert Transports.assertNotTransportThread("non-trivial nested loops over cluster state structures");
+        Set<String> pipelineIds = new HashSet<>();
+        Metadata metadata = state.metadata();
+        if (metadata == null) {
+            return pipelineIds;
+        }
+        IngestMetadata ingestMetadata = metadata.custom(IngestMetadata.TYPE);
+        if (ingestMetadata == null) {
+            return pipelineIds;
+        }
+        ingestMetadata.getPipelines().forEach((pipelineId, configuration) -> {
+            Map<String, Object> configMap = configuration.getConfigAsMap();
+            List<Map<String, Object>> processorConfigs = ConfigurationUtils.readList(null, null, configMap, PROCESSORS_KEY);
+            for (Map<String, Object> processorConfigWithKey : processorConfigs) {
+                for (Map.Entry<String, Object> entry : processorConfigWithKey.entrySet()) {
+                    addModelsAndPipelines(entry.getKey(), pipelineId, (Map<String, Object>) entry.getValue(), pam -> {
+                        if (ids.contains(pam.modelIdOrAlias)) {
+                            pipelineIds.add(pipelineId);
+                        }
+                    }, 0);
+                }
+            }
+        });
+        return pipelineIds;
+    }
+
     @SuppressWarnings("unchecked")
     private static void addModelsAndPipelines(
         String processorType,
@@ -146,7 +177,7 @@ private static void addModelsAndPipelines(
         if (processorType == null || processorDefinition == null) {
             return;
         }
-        if (TYPE.equals(processorType)) {
+        if (InferenceProcessorConstants.TYPE.equals(processorType)) {
             String modelId = (String) processorDefinition.get(MODEL_ID_RESULTS_FIELD);
             if (modelId != null) {
                 handler.accept(new PipelineAndModel(pipelineId, modelId));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/InferenceProcessorInfoExtractorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractorTests.java
similarity index 96%
rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/InferenceProcessorInfoExtractorTests.java
rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractorTests.java
index 488c974cc27bb..a5d823fc2144f 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/InferenceProcessorInfoExtractorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/InferenceProcessorInfoExtractorTests.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.ml.utils;
+package org.elasticsearch.xpack.core.ml.utils;
 
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -23,7 +23,6 @@
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfig;
-import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -217,15 +216,15 @@ private static Map<String, Object> forEachProcessorWithInference(String modelId)
     }
 
     private static Map<String, Object> inferenceProcessorForModel(String modelId) {
-        return Collections.singletonMap(InferenceProcessor.TYPE, new HashMap<>() {
+        return Collections.singletonMap(InferenceProcessorConstants.TYPE, new HashMap<>() {
             {
                 put(InferenceResults.MODEL_ID_RESULTS_FIELD, modelId);
                 put(
-                    InferenceProcessor.INFERENCE_CONFIG,
+                    InferenceProcessorConstants.INFERENCE_CONFIG,
                     Collections.singletonMap(RegressionConfig.NAME.getPreferredName(), Collections.emptyMap())
                 );
-                put(InferenceProcessor.TARGET_FIELD, "new_field");
-                put(InferenceProcessor.FIELD_MAP, Collections.singletonMap("source", "dest"));
+                put(InferenceProcessorConstants.TARGET_FIELD, "new_field");
+                put(InferenceProcessorConstants.FIELD_MAP, Collections.singletonMap("source", "dest"));
             }
         });
     }
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java
index e16d5a96a65d4..419869c0c4a5e 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java
@@ -113,6 +113,13 @@ protected void deleteModel(String modelId) throws IOException {
         assertOkOrCreated(response);
     }
 
+    protected Response deleteModel(String modelId, String queryParams) throws IOException {
+        var request = new Request("DELETE", "_inference/" + modelId + "?" + queryParams);
+        var response = client().performRequest(request);
+        assertOkOrCreated(response);
+        return response;
+    }
+
     protected void deleteModel(String modelId, TaskType taskType) throws IOException {
         var request = new Request("DELETE", Strings.format("_inference/%s/%s", taskType, modelId));
         var response = client().performRequest(request);
@@ -124,6 +131,29 @@ protected Map<String, Object> putModel(String modelId, String modelConfig, TaskT
         return putRequest(endpoint, modelConfig);
     }
 
+    protected Map<String, Object> putPipeline(String pipelineId, String modelId) throws IOException {
+        String endpoint = Strings.format("_ingest/pipeline/%s", pipelineId);
+        String body = """
+            {
+              "description": "Test pipeline",
+              "processors": [
+                {
+                  "inference": {
+                    "model_id": "%s"
+                  }
+                }
+              ]
+            }
+            """.formatted(modelId);
+        return putRequest(endpoint, body);
+    }
+
+    protected void deletePipeline(String pipelineId) throws IOException {
+        var request = new Request("DELETE", Strings.format("_ingest/pipeline/%s", pipelineId));
+        var response = client().performRequest(request);
+        assertOkOrCreated(response);
+    }
+
     /**
      * Task type should be in modelConfig
      */
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
index 57693134ecd41..75e392b6d155f 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
@@ -9,6 +9,7 @@
 
 package org.elasticsearch.xpack.inference;
 
+import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.inference.TaskType;
@@ -115,4 +116,34 @@ public void testSkipValidationAndStart() throws IOException {
         // We would expect an error about the invalid API key if the validation occurred
         putModel("unvalidated", openAiConfigWithBadApiKey, TaskType.TEXT_EMBEDDING);
     }
+
+    public void testDeleteEndpointWhileReferencedByPipeline() throws IOException {
+        String endpointId = "endpoint_referenced_by_pipeline";
+        putModel(endpointId, mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING);
+        var pipelineId = "pipeline_referencing_model";
+        putPipeline(pipelineId, endpointId);
+
+        {
+            var e = expectThrows(ResponseException.class, () -> deleteModel(endpointId));
+            assertThat(
+                e.getMessage(),
+                containsString(
+                    "Inference endpoint endpoint_referenced_by_pipeline is referenced by pipelines and cannot be deleted. "
+                        + "Use `force` to delete it anyway, or use `dry_run` to list the pipelines that reference it."
+                )
+            );
+        }
+        {
+            var response = deleteModel(endpointId, "dry_run=true");
+            var entityString = EntityUtils.toString(response.getEntity());
+            assertThat(entityString, containsString(pipelineId));
+            assertThat(entityString, containsString("\"acknowledged\":false"));
+        }
+        {
+            var response = deleteModel(endpointId, "force=true");
+            var entityString = EntityUtils.toString(response.getEntity());
+            assertThat(entityString, containsString("\"acknowledged\":true"));
+        }
+        deletePipeline(pipelineId);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
index 108da3beef789..9a9b11fd1400e 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
@@ -38,12 +38,12 @@
 import org.elasticsearch.threadpool.ScalingExecutorBuilder;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
-import org.elasticsearch.xpack.core.inference.action.DeleteInferenceModelAction;
+import org.elasticsearch.xpack.core.inference.action.DeleteInferenceEndpointAction;
 import org.elasticsearch.xpack.core.inference.action.GetInferenceDiagnosticsAction;
 import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.action.PutInferenceModelAction;
-import org.elasticsearch.xpack.inference.action.TransportDeleteInferenceModelAction;
+import org.elasticsearch.xpack.inference.action.TransportDeleteInferenceEndpointAction;
 import org.elasticsearch.xpack.inference.action.TransportGetInferenceDiagnosticsAction;
 import org.elasticsearch.xpack.inference.action.TransportGetInferenceModelAction;
 import org.elasticsearch.xpack.inference.action.TransportInferenceAction;
@@ -60,7 +60,7 @@
 import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
 import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder;
 import org.elasticsearch.xpack.inference.registry.ModelRegistry;
-import org.elasticsearch.xpack.inference.rest.RestDeleteInferenceModelAction;
+import org.elasticsearch.xpack.inference.rest.RestDeleteInferenceEndpointAction;
 import org.elasticsearch.xpack.inference.rest.RestGetInferenceDiagnosticsAction;
 import org.elasticsearch.xpack.inference.rest.RestGetInferenceModelAction;
 import org.elasticsearch.xpack.inference.rest.RestInferenceAction;
@@ -125,7 +125,7 @@ public InferencePlugin(Settings settings) {
             new ActionHandler<>(InferenceAction.INSTANCE, TransportInferenceAction.class),
             new ActionHandler<>(GetInferenceModelAction.INSTANCE, TransportGetInferenceModelAction.class),
             new ActionHandler<>(PutInferenceModelAction.INSTANCE, TransportPutInferenceModelAction.class),
-            new ActionHandler<>(DeleteInferenceModelAction.INSTANCE, TransportDeleteInferenceModelAction.class),
+            new ActionHandler<>(DeleteInferenceEndpointAction.INSTANCE, TransportDeleteInferenceEndpointAction.class),
             new ActionHandler<>(XPackUsageFeatureAction.INFERENCE, TransportInferenceUsageAction.class),
             new ActionHandler<>(GetInferenceDiagnosticsAction.INSTANCE, TransportGetInferenceDiagnosticsAction.class)
         );
@@ -147,7 +147,7 @@ public List<RestHandler> getRestHandlers(
             new RestInferenceAction(),
             new RestGetInferenceModelAction(),
             new RestPutInferenceModelAction(),
-            new RestDeleteInferenceModelAction(),
+            new RestDeleteInferenceEndpointAction(),
             new RestGetInferenceDiagnosticsAction()
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
new file mode 100644
index 0000000000000..5ebca2bc512a0
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.SubscribableListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.inference.InferenceServiceRegistry;
+import org.elasticsearch.ingest.IngestMetadata;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.inference.action.DeleteInferenceEndpointAction;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor;
+import org.elasticsearch.xpack.inference.common.InferenceExceptions;
+import org.elasticsearch.xpack.inference.registry.ModelRegistry;
+
+import java.util.Set;
+
+public class TransportDeleteInferenceEndpointAction extends AcknowledgedTransportMasterNodeAction<DeleteInferenceEndpointAction.Request> {
+
+    private final ModelRegistry modelRegistry;
+    private final InferenceServiceRegistry serviceRegistry;
+    private static final Logger logger = LogManager.getLogger(TransportDeleteInferenceEndpointAction.class);
+
+    @Inject
+    public TransportDeleteInferenceEndpointAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        ModelRegistry modelRegistry,
+        InferenceServiceRegistry serviceRegistry
+    ) {
+        super(
+            DeleteInferenceEndpointAction.NAME,
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            DeleteInferenceEndpointAction.Request::new,
+            indexNameExpressionResolver,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
+        this.modelRegistry = modelRegistry;
+        this.serviceRegistry = serviceRegistry;
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        DeleteInferenceEndpointAction.Request request,
+        ClusterState state,
+        ActionListener<AcknowledgedResponse> masterListener
+    ) {
+        SubscribableListener.<ModelRegistry.UnparsedModel>newForked(modelConfigListener -> {
+            // Get the model from the registry
+
+            modelRegistry.getModel(request.getInferenceEndpointId(), modelConfigListener);
+        }).<Boolean>andThen((listener, unparsedModel) -> {
+            // Validate the request & issue the stop request to the service
+
+            if (request.getTaskType().isAnyOrSame(unparsedModel.taskType()) == false) {
+                // specific task type in request does not match the models
+                listener.onFailure(InferenceExceptions.mismatchedTaskTypeException(request.getTaskType(), unparsedModel.taskType()));
+                return;
+            }
+
+            if (request.isDryRun()) {
+                masterListener.onResponse(
+                    new DeleteInferenceEndpointAction.Response(
+                        false,
+                        InferenceProcessorInfoExtractor.pipelineIdsForResource(state, Set.of(request.getInferenceEndpointId()))
+                    )
+                );
+                return;
+            } else if (request.isForceDelete() == false
+                && endpointIsReferencedInPipelines(state, request.getInferenceEndpointId(), listener)) {
+                    return;
+                }
+
+            var service = serviceRegistry.getService(unparsedModel.service());
+            if (service.isPresent()) {
+                service.get().stop(request.getInferenceEndpointId(), listener);
+            } else {
+                listener.onFailure(
+                    new ElasticsearchStatusException(
+                        "No service found for this inference endpoint " + request.getInferenceEndpointId(),
+                        RestStatus.NOT_FOUND
+                    )
+                );
+            }
+        }).<Boolean>andThen((listener, didStop) -> {
+            if (didStop) {
+                modelRegistry.deleteModel(request.getInferenceEndpointId(), listener);
+            } else {
+                listener.onFailure(
+                    new ElasticsearchStatusException(
+                        "Failed to stop inference endpoint " + request.getInferenceEndpointId(),
+                        RestStatus.INTERNAL_SERVER_ERROR
+                    )
+                );
+            }
+        })
+            .addListener(
+                masterListener.delegateFailure((l3, didDeleteModel) -> masterListener.onResponse(AcknowledgedResponse.of(didDeleteModel)))
+            );
+    }
+
+    private static boolean endpointIsReferencedInPipelines(
+        final ClusterState state,
+        final String inferenceEndpointId,
+        ActionListener<Boolean> listener
+    ) {
+        Metadata metadata = state.getMetadata();
+        if (metadata == null) {
+            listener.onFailure(
+                new ElasticsearchStatusException(
+                    " Could not determine if the endpoint is referenced in a pipeline as cluster state metadata was unexpectedly null. "
+                        + "Use `force` to delete it anyway",
+                    RestStatus.INTERNAL_SERVER_ERROR
+                )
+            );
+            // Unsure why the ClusterState metadata would ever be null, but in this case it seems safer to assume the endpoint is referenced
+            return true;
+        }
+        IngestMetadata ingestMetadata = metadata.custom(IngestMetadata.TYPE);
+        if (ingestMetadata == null) {
+            logger.debug("No ingest metadata found in cluster state while attempting to delete inference endpoint");
+        } else {
+            Set<String> modelIdsReferencedByPipelines = InferenceProcessorInfoExtractor.getModelIdsFromInferenceProcessors(ingestMetadata);
+            if (modelIdsReferencedByPipelines.contains(inferenceEndpointId)) {
+                listener.onFailure(
+                    new ElasticsearchStatusException(
+                        "Inference endpoint "
+                            + inferenceEndpointId
+                            + " is referenced by pipelines and cannot be deleted. "
+                            + "Use `force` to delete it anyway, or use `dry_run` to list the pipelines that reference it.",
+                        RestStatus.CONFLICT
+                    )
+                );
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(DeleteInferenceEndpointAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE);
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java
deleted file mode 100644
index be86cc971fb2a..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceModelAction.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.action;
-
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.SubscribableListener;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.inference.InferenceServiceRegistry;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.inference.action.DeleteInferenceModelAction;
-import org.elasticsearch.xpack.inference.common.InferenceExceptions;
-import org.elasticsearch.xpack.inference.registry.ModelRegistry;
-
-public class TransportDeleteInferenceModelAction extends AcknowledgedTransportMasterNodeAction<DeleteInferenceModelAction.Request> {
-
-    private final ModelRegistry modelRegistry;
-    private final InferenceServiceRegistry serviceRegistry;
-
-    @Inject
-    public TransportDeleteInferenceModelAction(
-        TransportService transportService,
-        ClusterService clusterService,
-        ThreadPool threadPool,
-        ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        ModelRegistry modelRegistry,
-        InferenceServiceRegistry serviceRegistry
-    ) {
-        super(
-            DeleteInferenceModelAction.NAME,
-            transportService,
-            clusterService,
-            threadPool,
-            actionFilters,
-            DeleteInferenceModelAction.Request::new,
-            indexNameExpressionResolver,
-            EsExecutors.DIRECT_EXECUTOR_SERVICE
-        );
-        this.modelRegistry = modelRegistry;
-        this.serviceRegistry = serviceRegistry;
-    }
-
-    @Override
-    protected void masterOperation(
-        Task task,
-        DeleteInferenceModelAction.Request request,
-        ClusterState state,
-        ActionListener<AcknowledgedResponse> listener
-    ) {
-        SubscribableListener.<ModelRegistry.UnparsedModel>newForked(modelConfigListener -> {
-            modelRegistry.getModel(request.getInferenceEntityId(), modelConfigListener);
-        }).<Boolean>andThen((l1, unparsedModel) -> {
-
-            if (request.getTaskType().isAnyOrSame(unparsedModel.taskType()) == false) {
-                // specific task type in request does not match the models
-                l1.onFailure(InferenceExceptions.mismatchedTaskTypeException(request.getTaskType(), unparsedModel.taskType()));
-                return;
-            }
-            var service = serviceRegistry.getService(unparsedModel.service());
-            if (service.isPresent()) {
-                service.get().stop(request.getInferenceEntityId(), l1);
-            } else {
-                l1.onFailure(
-                    new ElasticsearchStatusException("No service found for model " + request.getInferenceEntityId(), RestStatus.NOT_FOUND)
-                );
-            }
-        }).<Boolean>andThen((l2, didStop) -> {
-            if (didStop) {
-                modelRegistry.deleteModel(request.getInferenceEntityId(), l2);
-            } else {
-                l2.onFailure(
-                    new ElasticsearchStatusException(
-                        "Failed to stop model " + request.getInferenceEntityId(),
-                        RestStatus.INTERNAL_SERVER_ERROR
-                    )
-                );
-            }
-        }).addListener(listener.delegateFailure((l3, didDeleteModel) -> listener.onResponse(AcknowledgedResponse.of(didDeleteModel))));
-    }
-
-    @Override
-    protected ClusterBlockException checkBlock(DeleteInferenceModelAction.Request request, ClusterState state) {
-        return state.blocks().globalBlockedException(ClusterBlockLevel.WRITE);
-    }
-
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceEndpointAction.java
similarity index 75%
rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceModelAction.java
rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceEndpointAction.java
index 985a3c5045c3a..25280b8acf92d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceModelAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceEndpointAction.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.inference.action.DeleteInferenceModelAction;
+import org.elasticsearch.xpack.core.inference.action.DeleteInferenceEndpointAction;
 
 import java.util.List;
 
@@ -25,7 +25,10 @@
 import static org.elasticsearch.xpack.inference.rest.Paths.TASK_TYPE_OR_INFERENCE_ID;
 
 @ServerlessScope(Scope.PUBLIC)
-public class RestDeleteInferenceModelAction extends BaseRestHandler {
+public class RestDeleteInferenceEndpointAction extends BaseRestHandler {
+
+    private String FORCE_DELETE_NAME = "force";
+    private String DRY_RUN_NAME = "dry_run";
 
     @Override
     public String getName() {
@@ -41,6 +44,9 @@ public List<Route> routes() {
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
         String inferenceEntityId;
         TaskType taskType;
+        boolean forceDelete = false;
+        boolean dryRun = false;
+
         if (restRequest.hasParam(INFERENCE_ID)) {
             inferenceEntityId = restRequest.param(INFERENCE_ID);
             taskType = TaskType.fromStringOrStatusException(restRequest.param(TASK_TYPE_OR_INFERENCE_ID));
@@ -49,7 +55,11 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
             taskType = TaskType.ANY;
         }
 
-        var request = new DeleteInferenceModelAction.Request(inferenceEntityId, taskType);
-        return channel -> client.execute(DeleteInferenceModelAction.INSTANCE, request, new RestToXContentListener<>(channel));
+        forceDelete = restRequest.paramAsBoolean(FORCE_DELETE_NAME, false);
+
+        dryRun = restRequest.paramAsBoolean(DRY_RUN_NAME, false);
+
+        var request = new DeleteInferenceEndpointAction.Request(inferenceEntityId, taskType, forceDelete, dryRun);
+        return channel -> client.execute(DeleteInferenceEndpointAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureResetIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureResetIT.java
index 7158e494bee68..19dad8db8ef01 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureResetIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureResetIT.java
@@ -47,6 +47,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor.countInferenceProcessors;
 import static org.elasticsearch.xpack.ml.integration.ClassificationIT.KEYWORD_FIELD;
 import static org.elasticsearch.xpack.ml.integration.MlNativeDataFrameAnalyticsIntegTestCase.buildAnalytics;
 import static org.elasticsearch.xpack.ml.integration.PyTorchModelIT.BASE_64_ENCODED_MODEL;
@@ -55,7 +56,6 @@
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.getDataCounts;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.indexDocs;
-import static org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor.countInferenceProcessors;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index 48cf92f4cab10..6fdc4e73e184f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -475,7 +475,7 @@
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX;
 import static org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX;
-import static org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor.countInferenceProcessors;
+import static org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor.countInferenceProcessors;
 
 public class MachineLearning extends Plugin
     implements
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java
index 168a81ba554dc..1be5a765d1735 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java
@@ -42,9 +42,9 @@
 import org.elasticsearch.xpack.core.ml.inference.ModelAliasMetadata;
 import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentMetadata;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
-import org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor;
 
 import java.util.ArrayList;
 import java.util.HashMap;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java
index 78ac4bc11d17a..2a4b3ea70a57f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java
@@ -33,8 +33,8 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.DeleteTrainedModelAliasAction;
 import org.elasticsearch.xpack.core.ml.inference.ModelAliasMetadata;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
-import org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor;
 
 import java.util.HashMap;
 import java.util.Locale;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java
index eeab3d461661b..12a8e7aadee46 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsAction.java
@@ -74,7 +74,7 @@
 
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
-import static org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor.pipelineIdsByResource;
+import static org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor.pipelineIdsByResource;
 
 public class TransportGetTrainedModelsStatsAction extends TransportAction<
     GetTrainedModelsStatsAction.Request,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
index 10027cbd0d6bc..dc95d548c5f1b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
@@ -36,10 +36,10 @@
 import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentMetadata;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor;
 import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentClusterService;
 import org.elasticsearch.xpack.ml.inference.deployment.TrainedModelDeploymentTask;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
-import org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor;
 
 import java.util.List;
 import java.util.Objects;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
index dab66f887b9a3..6b14e60c00247 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
@@ -55,9 +55,10 @@
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ZeroShotClassificationConfigUpdate;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorConstants;
+import org.elasticsearch.xpack.core.ml.utils.InferenceProcessorInfoExtractor;
 import org.elasticsearch.xpack.ml.inference.loadingservice.LocalModel;
 import org.elasticsearch.xpack.ml.notifications.InferenceAuditor;
-import org.elasticsearch.xpack.ml.utils.InferenceProcessorInfoExtractor;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -85,15 +86,15 @@ public class InferenceProcessor extends AbstractProcessor {
         Setting.Property.NodeScope
     );
 
-    public static final String TYPE = "inference";
+    public static final String TYPE = InferenceProcessorConstants.TYPE;
     public static final String MODEL_ID = "model_id";
-    public static final String INFERENCE_CONFIG = "inference_config";
+    public static final String INFERENCE_CONFIG = InferenceProcessorConstants.INFERENCE_CONFIG;
     public static final String IGNORE_MISSING = "ignore_missing";
 
     // target field style mappings
-    public static final String TARGET_FIELD = "target_field";
+    public static final String TARGET_FIELD = InferenceProcessorConstants.TARGET_FIELD;
     public static final String FIELD_MAPPINGS = "field_mappings";
-    public static final String FIELD_MAP = "field_map";
+    public static final String FIELD_MAP = InferenceProcessorConstants.FIELD_MAP;
     private static final String DEFAULT_TARGET_FIELD = "ml.inference";
 
     // input field config

From 2c0ad093ef5bc2cdcc8962d22239a6d55d41c75d Mon Sep 17 00:00:00 2001
From: David Turner <david.turner@elastic.co>
Date: Fri, 31 May 2024 15:54:11 +0100
Subject: [PATCH 083/208] Fix up misc master-node timeouts in xpack/plugin
 (#109232)

More simple cases of #107984.
---
 ...ncySimulatingBlobStoreRepositoryTests.java |  1 +
 ...bstractFrozenAutoscalingIntegTestCase.java |  1 +
 .../DataTierAllocationDeciderIT.java          |  5 +-
 ...StreamLifecycleUsageTransportActionIT.java |  2 +-
 .../protocol/xpack/XPackUsageRequest.java     |  5 +-
 .../protocol/xpack/frozen/FreezeRequest.java  |  5 +-
 .../action/TransportXPackUsageAction.java     |  2 +-
 .../core/action/XPackUsageRequestBuilder.java | 26 -----
 .../xpack/core/ilm/MountSnapshotStep.java     |  2 +-
 .../rest/action/RestXPackUsageAction.java     | 34 +++----
 .../MountSearchableSnapshotRequest.java       |  6 +-
 .../index/engine/frozen/FrozenIndexIT.java    | 42 +++++++--
 .../frozen/FrozenIndexRecoveryTests.java      |  5 +-
 .../index/engine/frozen/FrozenIndexTests.java | 94 ++++++++++++++-----
 .../rest/action/RestFreezeIndexAction.java    |  8 +-
 .../monitoring/integration/MonitoringIT.java  |  9 +-
 .../cluster/ClusterStatsCollector.java        | 11 ++-
 .../cluster/ClusterStatsCollectorTests.java   |  4 +-
 .../lucene/bwc/ArchiveLicenseIntegTests.java  |  6 +-
 .../BaseSearchableSnapshotsIntegTestCase.java |  1 +
 .../ClusterStateApplierOrderingTests.java     |  1 +
 .../FrozenSearchableSnapshotsIntegTests.java  |  3 +
 ...pshotsCanMatchOnCoordinatorIntegTests.java |  3 +
 .../SearchableSnapshotsIntegTests.java        |  7 ++
 .../SearchableSnapshotsLicenseIntegTests.java |  2 +
 ...eSnapshotsSettingValidationIntegTests.java |  1 +
 ...hableSnapshotsSystemIndicesIntegTests.java |  1 +
 ...ableSnapshotsUuidValidationIntegTests.java |  1 +
 .../AllocationFilteringIntegTests.java        |  1 +
 ...chableSnapshotDiskThresholdIntegTests.java |  1 +
 ...archableSnapshotsPrewarmingIntegTests.java |  1 +
 ...tiallyCachedShardAllocationIntegTests.java |  1 +
 .../MountSearchableSnapshotRequestTests.java  | 12 +++
 .../authc/esnative/NativeRealmIntegTests.java |  5 +-
 .../plugin/freeze/FreezeIndexPlugin.java      | 11 ++-
 35 files changed, 216 insertions(+), 104 deletions(-)
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequestBuilder.java

diff --git a/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java b/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java
index 6d49971df4f0d..b2eca6752712b 100644
--- a/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java
+++ b/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java
@@ -123,6 +123,7 @@ public void testRetrieveSnapshots() throws Exception {
 
         logger.info("--> mount snapshot");
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             "test-idx",
             repositoryName,
             si.snapshotId().getName(),
diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java
index 74c027a0afbd2..d907a98255180 100644
--- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java
+++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java
@@ -97,6 +97,7 @@ protected void createAndMountIndex() throws InterruptedException, java.util.conc
         assertThat(total.storage(), equalTo(ByteSizeValue.ZERO));
 
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             restoredIndexName,
             fsRepoName,
             snapshotInfo.snapshotId().getName(),
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java
index 6fe63548a1430..add11d373b401 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java
@@ -25,8 +25,9 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
+import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 import org.elasticsearch.xpack.core.datatiers.DataTiersFeatureSetUsage;
 import org.junit.Before;
@@ -417,7 +418,7 @@ private void updatePreference(String tier) {
     }
 
     private DataTiersFeatureSetUsage getUsage() {
-        XPackUsageResponse usages = new XPackUsageRequestBuilder(client()).get();
+        XPackUsageResponse usages = safeGet(client().execute(XPackUsageAction.INSTANCE, new XPackUsageRequest(SAFE_AWAIT_TIMEOUT)));
         return usages.getUsages()
             .stream()
             .filter(u -> u instanceof DataTiersFeatureSetUsage)
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportActionIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportActionIT.java
index c1b4a4bf27890..996bc0eff5c1c 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportActionIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportActionIT.java
@@ -169,7 +169,7 @@ private void assertUsageResults(
         double averageRetention,
         boolean defaultRolloverUsed
     ) throws Exception {
-        XPackUsageFeatureResponse response = client().execute(DATA_STREAM_LIFECYCLE, new XPackUsageRequest()).get();
+        XPackUsageFeatureResponse response = safeGet(client().execute(DATA_STREAM_LIFECYCLE, new XPackUsageRequest(SAFE_AWAIT_TIMEOUT)));
         XContentBuilder builder = XContentFactory.jsonBuilder();
         builder = response.getUsage().toXContent(builder, ToXContent.EMPTY_PARAMS);
         Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageRequest.java
index e5fbc9e07955c..1a308c83d0d59 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackUsageRequest.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -18,8 +19,8 @@
 
 public class XPackUsageRequest extends MasterNodeRequest<XPackUsageRequest> {
 
-    public XPackUsageRequest() {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+    public XPackUsageRequest(TimeValue masterNodeTimeout) {
+        super(masterNodeTimeout);
     }
 
     public XPackUsageRequest(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/frozen/FreezeRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/frozen/FreezeRequest.java
index d1d04088dcdd7..9a0b807105c91 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/frozen/FreezeRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/frozen/FreezeRequest.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 
@@ -26,8 +27,8 @@ public class FreezeRequest extends AcknowledgedRequest<FreezeRequest> implements
     private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
     private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
 
-    public FreezeRequest(String... indices) {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT);
+    public FreezeRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout, String... indices) {
+        super(masterNodeTimeout, ackTimeout);
         this.indices = indices;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
index 8eddfecf0c92c..e9be5a62c8eb6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
@@ -69,7 +69,7 @@ protected void masterOperation(Task task, XPackUsageRequest request, ClusterStat
             @Override
             protected void doRun() {
                 if (responses.size() < usageActions().size()) {
-                    final var childRequest = new XPackUsageRequest();
+                    final var childRequest = new XPackUsageRequest(request.masterNodeTimeout());
                     childRequest.setParentTask(request.getParentTask());
                     client.executeLocally(
                         usageActions.get(responses.size()),
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequestBuilder.java
deleted file mode 100644
index 586033e4c8f41..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageRequestBuilder.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.core.action;
-
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
-import org.elasticsearch.client.internal.ElasticsearchClient;
-import org.elasticsearch.protocol.xpack.XPackUsageRequest;
-
-public class XPackUsageRequestBuilder extends MasterNodeOperationRequestBuilder<
-    XPackUsageRequest,
-    XPackUsageResponse,
-    XPackUsageRequestBuilder> {
-
-    public XPackUsageRequestBuilder(ElasticsearchClient client) {
-        this(client, XPackUsageAction.INSTANCE);
-    }
-
-    public XPackUsageRequestBuilder(ElasticsearchClient client, ActionType<XPackUsageResponse> action) {
-        super(client, action, new XPackUsageRequest());
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java
index 96f280b4e03c9..aac4d74144e95 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java
@@ -142,6 +142,7 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl
         overrideTierPreference(this.getKey().phase()).ifPresent(override -> settingsBuilder.put(DataTier.TIER_PREFERENCE, override));
 
         final MountSearchableSnapshotRequest mountSearchableSnapshotRequest = new MountSearchableSnapshotRequest(
+            TimeValue.MAX_VALUE,
             mountedIndexName,
             snapshotRepository,
             snapshotName,
@@ -153,7 +154,6 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl
             false,
             storageType
         );
-        mountSearchableSnapshotRequest.masterNodeTimeout(TimeValue.MAX_VALUE);
         getClient().execute(
             MountSearchableSnapshotAction.INSTANCE,
             mountSearchableSnapshotRequest,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackUsageAction.java
index 59a2cf3c936db..19c4ad17ca4d9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackUsageAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackUsageAction.java
@@ -7,18 +7,18 @@
 package org.elasticsearch.xpack.core.rest.action;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.http.HttpChannel;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
+import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 
 import java.io.IOException;
@@ -26,7 +26,6 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.OK;
-import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 @ServerlessScope(Scope.INTERNAL)
 public class RestXPackUsageAction extends BaseRestHandler {
@@ -43,20 +42,21 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        final TimeValue masterTimeout = getMasterNodeTimeout(request);
-        final HttpChannel httpChannel = request.getHttpChannel();
-        return channel -> new XPackUsageRequestBuilder(new RestCancellableNodeClient(client, httpChannel)).setMasterNodeTimeout(
-            masterTimeout
-        ).execute(new RestBuilderListener<>(channel) {
-            @Override
-            public RestResponse buildResponse(XPackUsageResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                for (XPackFeatureSet.Usage usage : response.getUsages()) {
-                    builder.field(usage.name(), usage);
+        final var usageRequest = new XPackUsageRequest(RestUtils.getMasterNodeTimeout(request));
+        return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
+            XPackUsageAction.INSTANCE,
+            usageRequest,
+            new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(XPackUsageResponse response, XContentBuilder builder) throws Exception {
+                    builder.startObject();
+                    for (XPackFeatureSet.Usage usage : response.getUsages()) {
+                        builder.field(usage.name(), usage);
+                    }
+                    builder.endObject();
+                    return new RestResponse(OK, builder);
                 }
-                builder.endObject();
-                return new RestResponse(OK, builder);
             }
-        });
+        );
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java
index fba742e288032..d8e0fcd15cad6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java
@@ -16,7 +16,9 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -38,6 +40,7 @@ public class MountSearchableSnapshotRequest extends MasterNodeRequest<MountSearc
         "mount_searchable_snapshot",
         false,
         (a, request) -> new MountSearchableSnapshotRequest(
+            RestUtils.getMasterNodeTimeout(request),
             Objects.requireNonNullElse((String) a[1], (String) a[0]),
             Objects.requireNonNull(request.param("repository")),
             Objects.requireNonNull(request.param("snapshot")),
@@ -92,6 +95,7 @@ public class MountSearchableSnapshotRequest extends MasterNodeRequest<MountSearc
      * Constructs a new mount searchable snapshot request, restoring an index with the settings needed to make it a searchable snapshot.
      */
     public MountSearchableSnapshotRequest(
+        TimeValue masterNodeTimeout,
         String mountedIndexName,
         String repositoryName,
         String snapshotName,
@@ -101,7 +105,7 @@ public MountSearchableSnapshotRequest(
         boolean waitForCompletion,
         Storage storage
     ) {
-        super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT);
+        super(masterNodeTimeout);
         this.mountedIndexName = Objects.requireNonNull(mountedIndexName);
         this.repositoryName = Objects.requireNonNull(repositoryName);
         this.snapshotName = Objects.requireNonNull(snapshotName);
diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java
index 43fc8c63c654b..7b33fe670efd7 100644
--- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java
+++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexIT.java
@@ -90,7 +90,12 @@ public void testTimestampRangeRecalculatedOnStalePrimaryAllocation() throws IOEx
 
         assertThat(client().prepareDelete("index", indexResponse.getId()).get().status(), equalTo(RestStatus.OK));
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index").waitForActiveShards(ActiveShardCount.ONE)));
+        assertAcked(
+            client().execute(
+                FreezeIndexAction.INSTANCE,
+                new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index").waitForActiveShards(ActiveShardCount.ONE)
+            )
+        );
 
         assertThat(
             clusterAdmin().prepareState().get().getState().metadata().index("index").getTimestampRange(),
@@ -179,7 +184,9 @@ public void testTimestampFieldTypeExposedByAllIndicesServices() throws Exception
             assertNull(indicesService.getTimestampFieldType(index));
         }
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index")).actionGet()
+        );
         ensureGreen("index");
         for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) {
             final PlainActionFuture<DateFieldMapper.DateFieldType> timestampFieldTypeFuture = new PlainActionFuture<>();
@@ -193,7 +200,12 @@ public void testTimestampFieldTypeExposedByAllIndicesServices() throws Exception
             assertThat(timestampFieldTypeFuture.get().dateTimeFormatter().parseMillis(date), equalTo(1580817683000L));
         }
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index").setFreeze(false)).actionGet());
+        assertAcked(
+            client().execute(
+                FreezeIndexAction.INSTANCE,
+                new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index").setFreeze(false)
+            ).actionGet()
+        );
         ensureGreen("index");
         for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) {
             assertNull(indicesService.getTimestampFieldType(index));
@@ -227,7 +239,10 @@ public void testRetryPointInTime() throws Exception {
         for (int i = 0; i < numDocs; i++) {
             prepareIndex(indexName).setSource("created_date", "2011-02-02").get();
         }
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indexName))
+                .actionGet()
+        );
         final OpenPointInTimeRequest openPointInTimeRequest = new OpenPointInTimeRequest(indexName).indicesOptions(
             IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED
         ).keepAlive(TimeValue.timeValueMinutes(2));
@@ -254,7 +269,12 @@ public void testRetryPointInTime() throws Exception {
                 }
             );
         } finally {
-            assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName).setFreeze(false)).actionGet());
+            assertAcked(
+                client().execute(
+                    FreezeIndexAction.INSTANCE,
+                    new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indexName).setFreeze(false)
+                ).actionGet()
+            );
             client().execute(TransportClosePointInTimeAction.TYPE, new ClosePointInTimeRequest(pitId)).actionGet();
         }
     }
@@ -275,7 +295,12 @@ public void testPointInTimeWithDeletedIndices() {
             prepareIndex("index-2").setId(id).setSource("value", i).get();
         }
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index-1", "index-2")).actionGet());
+        assertAcked(
+            client().execute(
+                FreezeIndexAction.INSTANCE,
+                new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index-1", "index-2")
+            ).actionGet()
+        );
         final OpenPointInTimeRequest openPointInTimeRequest = new OpenPointInTimeRequest("index-*").indicesOptions(
             IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED
         ).keepAlive(TimeValue.timeValueMinutes(2));
@@ -311,7 +336,10 @@ public void testOpenPointInTimeWithNoIndexMatched() {
             String id = Integer.toString(i);
             prepareIndex("test-index").setId(id).setSource("value", i).get();
         }
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("test-index")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "test-index"))
+                .actionGet()
+        );
         // include the frozen indices
         {
             final OpenPointInTimeRequest openPointInTimeRequest = new OpenPointInTimeRequest("test-*").indicesOptions(
diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexRecoveryTests.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexRecoveryTests.java
index 8bc32f0171f29..89991e5f5551a 100644
--- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexRecoveryTests.java
+++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexRecoveryTests.java
@@ -74,7 +74,10 @@ public Settings onNodeStopped(String nodeName) throws Exception {
                 for (int i = 0; i < moreDocs; i++) {
                     client.prepareIndex(indexName).setSource("num", i).get();
                 }
-                assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet());
+                assertAcked(
+                    client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indexName))
+                        .actionGet()
+                );
                 return super.onNodeStopped(nodeName);
             }
         });
diff --git a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java
index d1db706562a37..92d042a98b16e 100644
--- a/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java
+++ b/x-pack/plugin/frozen-indices/src/internalClusterTest/java/org/elasticsearch/index/engine/frozen/FrozenIndexTests.java
@@ -97,7 +97,10 @@ public void testCloseFreezeAndOpen() throws Exception {
         prepareIndex(indexName).setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         prepareIndex(indexName).setId("2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         prepareIndex(indexName).setId("3").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indexName))
+                .actionGet()
+        );
         expectThrows(
             ClusterBlockException.class,
             prepareIndex(indexName).setId("4").setSource("field", "value").setRefreshPolicy(IMMEDIATE)
@@ -187,7 +190,10 @@ public void testSearchAndGetAPIsAreThrottled() throws IOException {
         for (int i = 0; i < 10; i++) {
             prepareIndex(indexName).setId("" + i).setSource("field", "foo bar baz").get();
         }
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indexName))
+                .actionGet()
+        );
         int numRequests = randomIntBetween(20, 50);
         int numRefreshes = 0;
         int numSearches = 0;
@@ -231,7 +237,9 @@ public void testFreezeAndUnfreeze() {
             // sometimes close it
             assertAcked(indicesAdmin().prepareClose("index").get());
         }
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index")).actionGet()
+        );
         {
             IndicesService indexServices = getInstanceFromNode(IndicesService.class);
             Index index = resolveIndex("index");
@@ -243,7 +251,12 @@ public void testFreezeAndUnfreeze() {
             assertEquals(0, shard.refreshStats().getTotal());
             assertThat(indexService.getMetadata().getTimestampRange(), sameInstance(IndexLongFieldRange.UNKNOWN));
         }
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index").setFreeze(false)).actionGet());
+        assertAcked(
+            client().execute(
+                FreezeIndexAction.INSTANCE,
+                new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index").setFreeze(false)
+            ).actionGet()
+        );
         {
             IndicesService indexServices = getInstanceFromNode(IndicesService.class);
             Index index = resolveIndex("index");
@@ -269,12 +282,15 @@ private void assertIndexFrozen(String idx) {
 
     public void testDoubleFreeze() {
         createIndex("test-idx", Settings.builder().put("index.number_of_shards", 2).build());
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("test-idx")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "test-idx"))
+                .actionGet()
+        );
         ResourceNotFoundException exception = expectThrows(
             ResourceNotFoundException.class,
             client().execute(
                 FreezeIndexAction.INSTANCE,
-                new FreezeRequest("test-idx").indicesOptions(
+                new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "test-idx").indicesOptions(
                     IndicesOptions.builder().wildcardOptions(IndicesOptions.WildcardOptions.builder().allowEmptyExpressions(false)).build()
                 )
             )
@@ -287,12 +303,15 @@ public void testUnfreezeClosedIndices() {
         prepareIndex("idx").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         createIndex("idx-closed", Settings.builder().put("index.number_of_shards", 1).build());
         prepareIndex("idx-closed").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("idx")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "idx")).actionGet()
+        );
         assertAcked(indicesAdmin().prepareClose("idx-closed").get());
         assertAcked(
             client().execute(
                 FreezeIndexAction.INSTANCE,
-                new FreezeRequest("idx*").setFreeze(false).indicesOptions(IndicesOptions.strictExpand())
+                new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "idx*").setFreeze(false)
+                    .indicesOptions(IndicesOptions.strictExpand())
             )
         );
         ClusterStateResponse stateResponse = clusterAdmin().prepareState().get();
@@ -307,7 +326,10 @@ public void testFreezePattern() {
         prepareIndex(indexName).setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         createIndex("test-idx-1", Settings.builder().put("index.number_of_shards", 1).build());
         prepareIndex("test-idx-1").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indexName))
+                .actionGet()
+        );
         assertIndexFrozen(indexName);
 
         IndicesStatsResponse index = indicesAdmin().prepareStats(indexName).clear().setRefresh(true).get();
@@ -316,7 +338,9 @@ public void testFreezePattern() {
         index = indicesAdmin().prepareStats(indexName).clear().setRefresh(true).get();
         assertEquals(1, index.getTotal().refresh.getTotal());
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("test*")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "test*")).actionGet()
+        );
         assertIndexFrozen(indexName);
         assertIndexFrozen("test-idx-1");
         index = indicesAdmin().prepareStats(indexName).clear().setRefresh(true).get();
@@ -361,7 +385,9 @@ public void testCanMatch() throws IOException {
             );
         }
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index")).actionGet()
+        );
         {
 
             IndicesService indexServices = getInstanceFromNode(IndicesService.class);
@@ -467,7 +493,9 @@ public void testCanMatch() throws IOException {
     public void testWriteToFrozenIndex() {
         createIndex("idx", Settings.builder().put("index.number_of_shards", 1).build());
         prepareIndex("idx").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("idx")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "idx")).actionGet()
+        );
         assertIndexFrozen("idx");
         expectThrows(ClusterBlockException.class, prepareIndex("idx").setId("2").setSource("field", "value").setRefreshPolicy(IMMEDIATE));
     }
@@ -479,7 +507,7 @@ public void testIgnoreUnavailable() {
         assertAcked(
             client().execute(
                 FreezeIndexAction.INSTANCE,
-                new FreezeRequest("idx*", "not_available").indicesOptions(
+                new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "idx*", "not_available").indicesOptions(
                     IndicesOptions.fromParameters(null, "true", null, null, IndicesOptions.strictExpandOpen())
                 )
             )
@@ -490,14 +518,16 @@ public void testIgnoreUnavailable() {
 
     public void testUnfreezeClosedIndex() {
         createIndex("idx", Settings.builder().put("index.number_of_shards", 1).build());
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("idx")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "idx")).actionGet()
+        );
         assertAcked(indicesAdmin().prepareClose("idx"));
         assertEquals(IndexMetadata.State.CLOSE, clusterAdmin().prepareState().get().getState().metadata().index("idx").getState());
         expectThrows(
             IndexNotFoundException.class,
             client().execute(
                 FreezeIndexAction.INSTANCE,
-                new FreezeRequest("id*").setFreeze(false)
+                new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "id*").setFreeze(false)
                     .indicesOptions(
                         IndicesOptions.builder()
                             .wildcardOptions(IndicesOptions.WildcardOptions.builder().allowEmptyExpressions(false))
@@ -506,7 +536,12 @@ public void testUnfreezeClosedIndex() {
             )
         );
         // we don't resolve to closed indices
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("idx").setFreeze(false)).actionGet());
+        assertAcked(
+            client().execute(
+                FreezeIndexAction.INSTANCE,
+                new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "idx").setFreeze(false)
+            ).actionGet()
+        );
         assertEquals(IndexMetadata.State.OPEN, clusterAdmin().prepareState().get().getState().metadata().index("idx").getState());
     }
 
@@ -517,7 +552,9 @@ public void testFreezeIndexIncreasesIndexSettingsVersion() {
 
         final long settingsVersion = clusterAdmin().prepareState().get().getState().metadata().index(index).getSettingsVersion();
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(index)).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, index)).actionGet()
+        );
         assertIndexFrozen(index);
         assertThat(
             clusterAdmin().prepareState().get().getState().metadata().index(index).getSettingsVersion(),
@@ -543,7 +580,10 @@ public void testFreezeEmptyIndexWithTranslogOps() throws Exception {
             assertThat(indexService.getShard(0).getLastKnownGlobalCheckpoint(), greaterThanOrEqualTo(nbNoOps - 1L));
         });
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indexName))
+                .actionGet()
+        );
         assertIndexFrozen(indexName);
     }
 
@@ -557,7 +597,10 @@ public void testRecoveryState() {
             assertThat(indexResponse.status(), is(RestStatus.CREATED));
         }
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indexName))
+                .actionGet()
+        );
         assertIndexFrozen(indexName);
 
         final IndexMetadata indexMetadata = clusterAdmin().prepareState().get().getState().metadata().index(indexName);
@@ -602,7 +645,10 @@ public void testTranslogStats() {
         );
         assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedOps));
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(indexName)).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, indexName))
+                .actionGet()
+        );
         assertIndexFrozen(indexName);
 
         IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
@@ -621,7 +667,9 @@ public void testComputesTimestampRangeFromMilliseconds() {
         prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-05T01:02:03.456Z").get();
         prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567Z").get();
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index")).actionGet()
+        );
 
         final IndexLongFieldRange timestampFieldRange = clusterAdmin().prepareState()
             .get()
@@ -653,7 +701,9 @@ public void testComputesTimestampRangeFromNanoseconds() throws IOException {
         prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-05T01:02:03.456789012Z").get();
         prepareIndex("index").setSource(DataStream.TIMESTAMP_FIELD_NAME, "2010-01-06T02:03:04.567890123Z").get();
 
-        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest("index")).actionGet());
+        assertAcked(
+            client().execute(FreezeIndexAction.INSTANCE, new FreezeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index")).actionGet()
+        );
 
         final IndexLongFieldRange timestampFieldRange = clusterAdmin().prepareState()
             .get()
diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java
index 8d987042f8d2d..0daf2d8a1ebf5 100644
--- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java
+++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java
@@ -63,9 +63,11 @@ public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBui
             });
         }
 
-        FreezeRequest freezeRequest = new FreezeRequest(Strings.splitStringByCommaToArray(request.param("index")));
-        freezeRequest.ackTimeout(getAckTimeout(request));
-        freezeRequest.masterNodeTimeout(getMasterNodeTimeout(request));
+        final var freezeRequest = new FreezeRequest(
+            getMasterNodeTimeout(request),
+            getAckTimeout(request),
+            Strings.splitStringByCommaToArray(request.param("index"))
+        );
         freezeRequest.indicesOptions(IndicesOptions.fromRequest(request, freezeRequest.indicesOptions()));
         String waitForActiveShards = request.param("wait_for_active_shards");
         if (waitForActiveShards != null) {
diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
index a0990330aecb7..350d91048ac97 100644
--- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
+++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
@@ -33,7 +34,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
+import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
@@ -427,8 +428,10 @@ public void disableMonitoring() throws Exception {
         }, 30L, TimeUnit.SECONDS);
     }
 
-    private boolean getMonitoringUsageExportersDefined() throws Exception {
-        final XPackUsageResponse usageResponse = new XPackUsageRequestBuilder(client()).execute().get();
+    private boolean getMonitoringUsageExportersDefined() {
+        final XPackUsageResponse usageResponse = safeGet(
+            client().execute(XPackUsageAction.INSTANCE, new XPackUsageRequest(SAFE_AWAIT_TIMEOUT))
+        );
         final Optional<MonitoringFeatureSetUsage> monitoringUsage = usageResponse.getUsages()
             .stream()
             .filter(usage -> usage instanceof MonitoringFeatureSetUsage)
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java
index 1d84235c3deeb..a5f3792d6df8d 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java
@@ -25,8 +25,9 @@
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
+import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.Collector;
 
@@ -85,8 +86,6 @@ protected boolean shouldCollect(final boolean isElectedMaster) {
 
     @Override
     protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node, final long interval, final ClusterState clusterState) {
-        final Supplier<List<XPackFeatureSet.Usage>> usageSupplier = () -> new XPackUsageRequestBuilder(client).get().getUsages();
-
         final ClusterStatsResponse clusterStats = client.admin().cluster().prepareClusterStats().setTimeout(getCollectionTimeout()).get();
         ensureNoTimeouts(getCollectionTimeout(), clusterStats);
 
@@ -94,7 +93,11 @@ protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node, fin
         final String clusterUuid = clusterUuid(clusterState);
         final String version = Build.current().version();
         final License license = licenseService.getLicense();
-        final List<XPackFeatureSet.Usage> xpackUsage = collect(usageSupplier);
+        final List<XPackFeatureSet.Usage> xpackUsage = collect(
+            () -> client.execute(XPackUsageAction.INSTANCE, new XPackUsageRequest(getCollectionTimeout()))
+                .actionGet(getCollectionTimeout())
+                .getUsages()
+        );
         final boolean apmIndicesExist = doAPMIndicesExist(clusterState);
         // if they have any other type of license, then they are either okay or already know
         final boolean clusterNeedsTLSEnabled = license != null
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java
index 504dbc65eac2a..d5aeebd63a9e5 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java
@@ -235,7 +235,7 @@ public void testDoCollect() throws Exception {
         @SuppressWarnings("unchecked")
         final ActionFuture<XPackUsageResponse> xPackUsageFuture = (ActionFuture<XPackUsageResponse>) mock(ActionFuture.class);
         when(client.execute(same(XPackUsageAction.INSTANCE), any(XPackUsageRequest.class))).thenReturn(xPackUsageFuture);
-        when(xPackUsageFuture.actionGet()).thenReturn(xPackUsageResponse);
+        when(xPackUsageFuture.actionGet(any(TimeValue.class))).thenReturn(xPackUsageResponse);
 
         final ClusterStatsCollector collector = new ClusterStatsCollector(
             settings.build(),
@@ -345,7 +345,7 @@ public void testDoCollectNoLicense() throws Exception {
             @SuppressWarnings("unchecked")
             final ActionFuture<XPackUsageResponse> xPackUsageFuture = (ActionFuture<XPackUsageResponse>) mock(ActionFuture.class);
             when(client.execute(same(XPackUsageAction.INSTANCE), any(XPackUsageRequest.class))).thenReturn(xPackUsageFuture);
-            when(xPackUsageFuture.actionGet()).thenReturn(xPackUsageResponse);
+            when(xPackUsageFuture.actionGet(any(TimeValue.class))).thenReturn(xPackUsageResponse);
         }
 
         final long interval = randomNonNegativeLong();
diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java
index 048e259eca977..a482cd2c364e2 100644
--- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java
+++ b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/ArchiveLicenseIntegTests.java
@@ -39,7 +39,9 @@
 public class ArchiveLicenseIntegTests extends AbstractArchiveTestCase {
 
     public void testFeatureUsage() throws Exception {
-        XPackUsageFeatureResponse usage = client().execute(XPackUsageFeatureAction.ARCHIVE, new XPackUsageRequest()).get();
+        XPackUsageFeatureResponse usage = safeGet(
+            client().execute(XPackUsageFeatureAction.ARCHIVE, new XPackUsageRequest(SAFE_AWAIT_TIMEOUT))
+        );
         assertThat(usage.getUsage(), instanceOf(ArchiveFeatureSetUsage.class));
         ArchiveFeatureSetUsage archiveUsage = (ArchiveFeatureSetUsage) usage.getUsage();
         assertEquals(0, archiveUsage.getNumberOfArchiveIndices());
@@ -50,7 +52,7 @@ public void testFeatureUsage() throws Exception {
         assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
         ensureGreen(indexName);
 
-        usage = client().execute(XPackUsageFeatureAction.ARCHIVE, new XPackUsageRequest()).get();
+        usage = safeGet(client().execute(XPackUsageFeatureAction.ARCHIVE, new XPackUsageRequest(SAFE_AWAIT_TIMEOUT)));
         assertThat(usage.getUsage(), instanceOf(ArchiveFeatureSetUsage.class));
         archiveUsage = (ArchiveFeatureSetUsage) usage.getUsage();
         assertEquals(1, archiveUsage.getNumberOfArchiveIndices());
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java
index ad5f57645aa84..f73890e50a3a2 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java
@@ -184,6 +184,7 @@ protected void mountSnapshot(
         final Storage storage
     ) throws Exception {
         final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             restoredIndexName,
             repositoryName,
             snapshotName,
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java
index 82ded22603ef1..ffedcb8f9ebd3 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java
@@ -65,6 +65,7 @@ public void testRepositoriesServiceClusterStateApplierIsCalledBeforeIndicesClust
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
 
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             restoredIndexName,
             fsRepoName,
             snapshotInfo.snapshotId().getName(),
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java
index be37cc2184d15..ec7ca2ae5b681 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java
@@ -225,6 +225,7 @@ public void testCreateAndRestorePartialSearchableSnapshot() throws Exception {
         statsWatcher.start();
 
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             restoredIndexName,
             fsRepoName,
             snapshotInfo.snapshotId().getName(),
@@ -453,6 +454,7 @@ public void testRequestCacheOnFrozen() throws Exception {
 
         Settings.Builder indexSettingsBuilder = Settings.builder().put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true);
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             "test-index",
             "repo",
             "snap",
@@ -565,6 +567,7 @@ public void testTierPreferenceCannotBeRemovedForFrozenIndex() throws Exception {
             .putNull(DataTier.TIER_PREFERENCE);
 
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             restoredIndexName,
             fsRepoName,
             snapshotInfo.snapshotId().getName(),
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
index d2b7de9353a0d..5204bdfcc78e6 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
@@ -152,6 +152,7 @@ public void testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQuerying
             .build();
 
         final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             searchableSnapshotIndexOutsideSearchRange,
             repositoryName,
             snapshotId.getName(),
@@ -422,6 +423,7 @@ public void testQueryPhaseIsExecutedInAnAvailableNodeWhenAllShardsCanBeSkipped()
             .build();
 
         final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             searchableSnapshotIndexOutsideSearchRange,
             repositoryName,
             snapshotId.getName(),
@@ -605,6 +607,7 @@ public void testSearchableSnapshotShardsThatHaveMatchingDataAreNotSkippedOnTheCo
             .build();
 
         final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             searchableSnapshotIndexWithinSearchRange,
             repositoryName,
             snapshotId.getName(),
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
index 50be307fe16a4..13e5833b133d5 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
@@ -209,6 +209,7 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception {
         }
 
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             restoredIndexName,
             fsRepoName,
             snapshotInfo.snapshotId().getName(),
@@ -398,6 +399,7 @@ public void testCanMountSnapshotTakenWhileConcurrentlyIndexing() throws Exceptio
 
         Settings.Builder indexSettingsBuilder = Settings.builder().put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true);
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             restoredIndexName,
             fsRepoName,
             snapshotName,
@@ -466,6 +468,7 @@ public void testMaxRestoreBytesPerSecIsUsed() throws Exception {
 
         logger.info("--> restoring index [{}] using rate limits [{}]", restoredIndexName, useRateLimits);
         final MountSearchableSnapshotRequest mount = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             restoredIndexName,
             repositoryName,
             snapshotName,
@@ -550,6 +553,7 @@ public void testMountedSnapshotHasNoReplicasByDefault() throws Exception {
             Settings.Builder indexSettingsBuilder = Settings.builder()
                 .put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true);
             final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 restoredIndexName,
                 fsRepoName,
                 snapshotName,
@@ -582,6 +586,7 @@ public void testMountedSnapshotHasNoReplicasByDefault() throws Exception {
                 .put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true)
                 .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicaCount);
             final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 restoredIndexName,
                 fsRepoName,
                 snapshotName,
@@ -613,6 +618,7 @@ public void testMountedSnapshotHasNoReplicasByDefault() throws Exception {
                 .put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true)
                 .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, replicaLimit == dataNodesCount ? "0-all" : "0-" + replicaLimit);
             final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 restoredIndexName,
                 fsRepoName,
                 snapshotName,
@@ -1191,6 +1197,7 @@ public void onFailure(Exception e) {
         ActionFuture<RestoreSnapshotResponse> response = null;
         try {
             final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 indexName,
                 repositoryName,
                 snapshotOne.getName(),
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java
index 1216db4b17608..73f12f98f8ca0 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java
@@ -64,6 +64,7 @@ public void createAndMountSearchableSnapshot() throws Exception {
         assertAcked(indicesAdmin().prepareDelete(indexName));
 
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             indexName,
             repoName,
             snapshotName,
@@ -92,6 +93,7 @@ public void createAndMountSearchableSnapshot() throws Exception {
 
     public void testMountRequiresLicense() {
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             indexName + "-extra",
             repoName,
             snapshotName,
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSettingValidationIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSettingValidationIntegTests.java
index fd6d1b4e79418..2bc9e31af1836 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSettingValidationIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSettingValidationIntegTests.java
@@ -36,6 +36,7 @@ public void createAndMountSearchableSnapshot() throws Exception {
         assertAcked(indicesAdmin().prepareDelete(indexName));
 
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             indexName,
             repoName,
             snapshotName,
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java
index 1c78d842ee4c9..ed03a8e898882 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java
@@ -76,6 +76,7 @@ private void executeTest(final String indexName, final String featureName, final
         }
 
         final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             indexName,
             repositoryName,
             snapshotName,
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsUuidValidationIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsUuidValidationIntegTests.java
index 42ff8cb2f053d..c352c37ccadf8 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsUuidValidationIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsUuidValidationIntegTests.java
@@ -97,6 +97,7 @@ public void testMountFailsIfSnapshotChanged() throws Exception {
         createFullSnapshot(fsRepoName, snapshotName);
 
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             restoredIndexName,
             fsRepoName,
             snapshotName,
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/AllocationFilteringIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/AllocationFilteringIntegTests.java
index 2839639c5776c..530b4fe937684 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/AllocationFilteringIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/AllocationFilteringIntegTests.java
@@ -62,6 +62,7 @@ private MountSearchableSnapshotRequest prepareMountRequest(
             .put(mountedIndexSettings.build());
 
         return new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             indexName,
             fsRepoName,
             snapshotName,
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java
index e4f9d530e83df..73e2e56b31ca5 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java
@@ -177,6 +177,7 @@ private void mountIndices(Collection<String> indices, String prefix, String repo
             client().execute(
                 MountSearchableSnapshotAction.INSTANCE,
                 new MountSearchableSnapshotRequest(
+                    TEST_REQUEST_TIMEOUT,
                     prefix + index,
                     repositoryName,
                     snapshotName,
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java
index 30c96b0a1ddda..42542b63c80d1 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java
@@ -215,6 +215,7 @@ public void testConcurrentPrewarming() throws Exception {
                     final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(
                         MountSearchableSnapshotAction.INSTANCE,
                         new MountSearchableSnapshotRequest(
+                            TEST_REQUEST_TIMEOUT,
                             indexName,
                             "repository",
                             "snapshot",
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java
index 28da64b4ceaec..7b372ddc53d80 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java
@@ -91,6 +91,7 @@ private MountSearchableSnapshotRequest prepareMountRequest() throws InterruptedE
             .put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true);
 
         return new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             indexName,
             fsRepoName,
             snapshotName,
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java
index 1f40596cbf1e6..7cfe6176a7437 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java
@@ -33,6 +33,7 @@ public class MountSearchableSnapshotRequestTests extends AbstractWireSerializing
 
     private MountSearchableSnapshotRequest randomState(MountSearchableSnapshotRequest instance) {
         return new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             randomBoolean() ? instance.mountedIndexName() : mutateString(instance.mountedIndexName()),
             randomBoolean() ? instance.repositoryName() : mutateString(instance.repositoryName()),
             randomBoolean() ? instance.snapshotName() : mutateString(instance.snapshotName()),
@@ -48,6 +49,7 @@ private MountSearchableSnapshotRequest randomState(MountSearchableSnapshotReques
     protected MountSearchableSnapshotRequest createTestInstance() {
         return randomState(
             new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 randomAlphaOfLength(5),
                 randomAlphaOfLength(5),
                 randomAlphaOfLength(5),
@@ -69,6 +71,7 @@ protected Writeable.Reader<MountSearchableSnapshotRequest> instanceReader() {
     protected MountSearchableSnapshotRequest mutateInstance(MountSearchableSnapshotRequest req) {
         return switch (randomInt(8)) {
             case 0 -> new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 mutateString(req.mountedIndexName()),
                 req.repositoryName(),
                 req.snapshotName(),
@@ -79,6 +82,7 @@ protected MountSearchableSnapshotRequest mutateInstance(MountSearchableSnapshotR
                 req.storage()
             ).masterNodeTimeout(req.masterNodeTimeout());
             case 1 -> new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 req.mountedIndexName(),
                 mutateString(req.repositoryName()),
                 req.snapshotName(),
@@ -89,6 +93,7 @@ protected MountSearchableSnapshotRequest mutateInstance(MountSearchableSnapshotR
                 req.storage()
             ).masterNodeTimeout(req.masterNodeTimeout());
             case 2 -> new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 req.mountedIndexName(),
                 req.repositoryName(),
                 mutateString(req.snapshotName()),
@@ -99,6 +104,7 @@ protected MountSearchableSnapshotRequest mutateInstance(MountSearchableSnapshotR
                 req.storage()
             ).masterNodeTimeout(req.masterNodeTimeout());
             case 3 -> new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 req.mountedIndexName(),
                 req.repositoryName(),
                 req.snapshotName(),
@@ -109,6 +115,7 @@ protected MountSearchableSnapshotRequest mutateInstance(MountSearchableSnapshotR
                 req.storage()
             ).masterNodeTimeout(req.masterNodeTimeout());
             case 4 -> new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 req.mountedIndexName(),
                 req.repositoryName(),
                 req.snapshotName(),
@@ -119,6 +126,7 @@ protected MountSearchableSnapshotRequest mutateInstance(MountSearchableSnapshotR
                 req.storage()
             ).masterNodeTimeout(req.masterNodeTimeout());
             case 5 -> new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 req.mountedIndexName(),
                 req.repositoryName(),
                 req.snapshotName(),
@@ -129,6 +137,7 @@ protected MountSearchableSnapshotRequest mutateInstance(MountSearchableSnapshotR
                 req.storage()
             ).masterNodeTimeout(req.masterNodeTimeout());
             case 6 -> new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 req.mountedIndexName(),
                 req.repositoryName(),
                 req.snapshotName(),
@@ -139,6 +148,7 @@ protected MountSearchableSnapshotRequest mutateInstance(MountSearchableSnapshotR
                 req.storage()
             ).masterNodeTimeout(req.masterNodeTimeout());
             case 7 -> new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 req.mountedIndexName(),
                 req.repositoryName(),
                 req.snapshotName(),
@@ -149,6 +159,7 @@ protected MountSearchableSnapshotRequest mutateInstance(MountSearchableSnapshotR
                 randomValueOtherThan(req.storage(), () -> randomFrom(MountSearchableSnapshotRequest.Storage.values()))
             ).masterNodeTimeout(req.masterNodeTimeout());
             default -> new MountSearchableSnapshotRequest(
+                TEST_REQUEST_TIMEOUT,
                 req.mountedIndexName(),
                 req.repositoryName(),
                 req.snapshotName(),
@@ -196,6 +207,7 @@ private static String[] mutateStringArray(String[] strings) {
 
     public void testForbidsCustomDataPath() {
         final ActionRequestValidationException validationException = new MountSearchableSnapshotRequest(
+            TEST_REQUEST_TIMEOUT,
             randomAlphaOfLength(5),
             randomAlphaOfLength(5),
             randomAlphaOfLength(5),
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
index ce4c8719f0642..8692c999d8b35 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotState;
@@ -29,7 +30,7 @@
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
+import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequestBuilder;
@@ -901,7 +902,7 @@ public void testRealmUsageStats() {
             preparePutUser("joe" + i, "s3krit-password", hasher, "superuser").get();
         }
 
-        XPackUsageResponse response = new XPackUsageRequestBuilder(client()).get();
+        XPackUsageResponse response = safeGet(client().execute(XPackUsageAction.INSTANCE, new XPackUsageRequest(SAFE_AWAIT_TIMEOUT)));
         Optional<XPackFeatureSet.Usage> securityUsage = response.getUsages()
             .stream()
             .filter(usage -> usage instanceof SecurityFeatureSetUsage)
diff --git a/x-pack/qa/freeze-plugin/src/main/java/org/elasticsearch/plugin/freeze/FreezeIndexPlugin.java b/x-pack/qa/freeze-plugin/src/main/java/org/elasticsearch/plugin/freeze/FreezeIndexPlugin.java
index 53c0088114424..af9a741b0aef1 100644
--- a/x-pack/qa/freeze-plugin/src/main/java/org/elasticsearch/plugin/freeze/FreezeIndexPlugin.java
+++ b/x-pack/qa/freeze-plugin/src/main/java/org/elasticsearch/plugin/freeze/FreezeIndexPlugin.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction;
 
-import java.io.IOException;
 import java.util.List;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -74,11 +73,13 @@ public List<Route> routes() {
         }
 
         @Override
-        protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
             boolean freeze = request.path().endsWith("/_freeze");
-            FreezeRequest freezeRequest = new FreezeRequest(Strings.splitStringByCommaToArray(request.param("index")));
-            freezeRequest.ackTimeout(getAckTimeout(request));
-            freezeRequest.masterNodeTimeout(getMasterNodeTimeout(request));
+            FreezeRequest freezeRequest = new FreezeRequest(
+                getMasterNodeTimeout(request),
+                getAckTimeout(request),
+                Strings.splitStringByCommaToArray(request.param("index"))
+            );
             freezeRequest.indicesOptions(IndicesOptions.fromRequest(request, freezeRequest.indicesOptions()));
             String waitForActiveShards = request.param("wait_for_active_shards");
             if (waitForActiveShards != null) {

From 95ce8984366c766f348300f962663ab6c9cda6de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?=
 <istvan.szabo@elastic.co>
Date: Fri, 31 May 2024 16:56:07 +0200
Subject: [PATCH 084/208] [DOCS] Adds docs to semantic text (#108311)

Co-authored-by: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com>
Co-authored-by: Mike Pellegrini <mike.pellegrini@elastic.co>
Co-authored-by: Kathleen DeRusso <kathleen.derusso@elastic.co>
---
 docs/reference/mapping/types.asciidoc         |   2 +-
 .../mapping/types/semantic-text.asciidoc      | 133 ++++++++-
 .../query-dsl/semantic-query.asciidoc         | 187 ++++++++++++
 .../query-dsl/special-queries.asciidoc        |   5 +
 .../semantic-search-semantic-text.asciidoc    | 267 ++++++++++++++++++
 .../search-your-data/semantic-search.asciidoc |   1 +
 6 files changed, 593 insertions(+), 2 deletions(-)
 create mode 100644 docs/reference/query-dsl/semantic-query.asciidoc
 create mode 100644 docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc

diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc
index 7ee1face25339..7e2e7083fa70b 100644
--- a/docs/reference/mapping/types.asciidoc
+++ b/docs/reference/mapping/types.asciidoc
@@ -75,7 +75,7 @@ markup. Used for identifying named entities.
 <<completion-suggester,`completion`>>:: Used for auto-complete suggestions.
 <<search-as-you-type,`search_as_you_type`>>:: `text`-like type for
 as-you-type completion.
-<<semantic-text, `semantic_text`>>:: 
+<<semantic-text, `semantic_text`>>:: Used for performing <<semantic-search,semantic search>>.
 <<token-count,`token_count`>>:: A count of tokens in a text.
 
 
diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc
index 51d7693db12aa..7fc1314ff888f 100644
--- a/docs/reference/mapping/types/semantic-text.asciidoc
+++ b/docs/reference/mapping/types/semantic-text.asciidoc
@@ -5,4 +5,135 @@
 <titleabbrev>Semantic text</titleabbrev>
 ++++
 
-The documentation page for the `semantic_text` field type.
\ No newline at end of file
+beta[]
+
+The `semantic_text` field type automatically generates embeddings for text
+content using an inference endpoint. 
+
+The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings.
+You can create the inference endpoint by using the <<put-inference-api>>.
+This field type and the <<query-dsl-semantic-query,`semantic` query>> type make it simpler to perform semantic search on your data.
+
+Using `semantic_text`, you won't need to specify how to generate embeddings for
+your data, or how to index it. The inference endpoint automatically determines
+the embedding generation, indexing, and query to use.
+
+[source,console]
+------------------------------------------------------------
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "inference_field": { 
+        "type": "semantic_text",
+        "inference_id": "my-elser-endpoint"
+      }
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+
+[discrete]
+[[semantic-text-params]]
+==== Parameters for `semantic_text` fields
+
+`inference_id`::
+(Required, string)  
+Inference endpoint that will be used to generate the embeddings for the field.
+Use the <<put-inference-api>> to create the endpoint.
+
+
+[discrete]
+[[infer-endpoint-validation]]
+==== {infer-cap} endpoint validation
+
+The `inference_id` will not be validated when the mapping is created, but when documents are ingested into the index.
+When the first document is indexed, the `inference_id` will be used to generate underlying indexing structures for the field.
+
+
+[discrete]
+[[auto-text-chunking]]
+==== Automatic text chunking
+
+{infer-cap} endpoints have a limit on the amount of text they can process.
+To allow for large amounts of text to be used in semantic search, `semantic_text` automatically generates smaller passages if needed, called _chunks_.
+
+Each chunk will include the text subpassage and the corresponding embedding generated from it.
+When querying, the individual passages will be automatically searched for each document, and the most relevant passage will be used to compute a score.
+
+
+[discrete]
+[[semantic-text-structure]]
+==== `semantic_text` structure
+
+Once a document is ingested, a `semantic_text` field will have the following structure:
+
+[source,console-result]
+------------------------------------------------------------
+"inference_field": {
+  "text": "these are not the droids you're looking for", <1>
+  "inference": {
+    "inference_id": "my-elser-endpoint", <2>
+    "model_settings": { <3>
+      "task_type": "sparse_embedding"
+    },
+    "chunks": [ <4>
+      {
+        "text": "these are not the droids you're looking for",
+        "embeddings": {
+          (...)
+        }
+      }
+    ]
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The field will become an object structure to accommodate both the original
+text and the inference results.
+<2> The `inference_id` used to generate the embeddings.
+<3> Model settings, including the task type and dimensions/similarity if
+applicable.
+<4> Inference results will be grouped in chunks, each with its corresponding
+text and embeddings.
+
+Refer to <<semantic-search-semantic-text,this tutorial>> to learn more about
+semantic search using `semantic_text` and the `semantic` query.
+
+
+[discrete]
+[[custom-indexing]]
+==== Customizing `semantic_text` indexing
+
+`semantic_text` uses defaults for indexing data based on the {infer} endpoint
+specified. It enables you to quickstart your semantic search by providing
+automatic {infer} and a dedicated query so you don't need to provide further
+details.
+
+In case you want to customize data indexing, use the
+<<sparse-vector,`sparse_vector`>> or <<dense-vector,`dense_vector`>> field
+types and create an ingest pipeline with an
+<<inference-processor, {infer} processor>> to generate the embeddings.
+<<semantic-search-inference,This tutorial>> walks you through the process.
+
+[discrete]
+[[update-script]]
+==== Updates to `semantic_text` fields
+
+Updates that use scripts are not supported when the index contains a `semantic_text` field.
+
+
+[discrete]
+[[copy-to-support]]
+==== `copy_to` support
+
+The `semantic_text` field type can be the target of
+<<copy-to,`copy_to` fields>>. This means you can use a single `semantic_text`
+field to collect the values of other fields for semantic search. Each value has
+its embeddings calculated separately; each field value is a separate set of chunk(s) in
+the resulting embeddings.
+
+This imposes a restriction on bulk updates to documents with `semantic_text`. 
+In bulk requests, all fields that are copied to a `semantic_text` field must have a value to ensure every embedding is calculated correctly.
diff --git a/docs/reference/query-dsl/semantic-query.asciidoc b/docs/reference/query-dsl/semantic-query.asciidoc
new file mode 100644
index 0000000000000..ccd94f0ecb77a
--- /dev/null
+++ b/docs/reference/query-dsl/semantic-query.asciidoc
@@ -0,0 +1,187 @@
+[[query-dsl-semantic-query]]
+=== Semantic query
+++++
+<titleabbrev>Semantic</titleabbrev>
+++++
+
+beta[]
+
+The `semantic` query type enables you to perform <<semantic-search,semantic search>> on data stored in a <<semantic-text,`semantic_text`>> field.
+
+
+[discrete]
+[[semantic-query-example]]
+==== Example request
+
+[source,console]
+------------------------------------------------------------
+GET my-index-000001/_search
+{
+  "query": {
+    "semantic": {
+      "field": "inference_field",
+      "query": "Best surfing places"
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+
+[discrete]
+[[semantic-query-params]]
+==== Top-level parameters for `semantic`
+
+field::
+(Required, string)
+The `semantic_text` field to perform the query on.
+
+query::
+(Required, string)
+The query text to be searched for on the field.
+
+
+Refer to <<semantic-search-semantic-text,this tutorial>> to learn more about semantic search using `semantic_text` and `semantic` query.
+
+[discrete]
+[[hybrid-search-semantic]]
+==== Hybrid search with the `semantic` query
+
+The `semantic` query can be used as a part of a hybrid search where the `semantic` query is combined with lexical queries.
+For example, the query below finds documents with the `title` field matching "mountain lake", and combines them with results from a semantic search on the field `title_semantic`, that is a `semantic_text` field.
+The combined documents are then scored, and the top 3 top scored documents are returned.
+
+[source,console]
+------------------------------------------------------------
+POST my-index/_search
+{
+  "size" : 3,
+  "query": {
+    "bool": {
+      "should": [
+        {
+          "match": {
+            "title": {
+              "query": "mountain lake",
+              "boost": 1
+            }
+          }
+        },
+        {
+          "semantic": {
+            "field": "title_semantic",
+            "query": "mountain lake",
+            "boost": 2
+          }
+        }
+      ]
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+You can also use semantic_text as part of <<rrf,Reciprocal Rank Fusion>> to make ranking relevant results easier:
+
+[source,console]
+------------------------------------------------------------
+GET my-index/_search
+{
+  "retriever": {
+    "rrf": {
+      "retrievers": [
+        {
+          "standard": {
+            "query": {
+              "term": {
+                "text": "shoes"
+              }
+            }
+          }
+        },
+        {
+          "semantic": {
+            "field": "semantic_field",
+            "query": "shoes"
+          }
+        }
+      ],
+      "rank_window_size": 50,
+      "rank_constant": 20
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+
+[discrete]
+[[advanced-search]]
+=== Advanced search on `semantic_text` fields
+
+The `semantic` query uses default settings for searching on `semantic_text` fields for ease of use.
+If you want to fine-tune a search on a `semantic_text` field, you need to know the task type used by the `inference_id` configured in `semantic_text`.
+You can find the task type using the <<get-inference-api>>, and check the `task_type` associated with the {infer} service.
+Depending on the `task_type`, use either the <<query-dsl-sparse-vector-query,`sparse_vector`>> or the <<query-dsl-knn-query,`knn`>> query for greater flexibility and customization.
+
+
+[discrete]
+[[search-sparse-inference]]
+==== Search with `sparse_embedding` inference
+
+When the {infer} endpoint uses a `sparse_embedding` model, you can use a <<query-dsl-sparse-vector-query,`sparse_vector` query>> on a <<semantic-text,`semantic_text`>> field in the following way:
+
+[source,console]
+------------------------------------------------------------
+GET test-index/_search
+{
+  "query": {
+    "nested": {
+      "path": "inference_field.inference.chunks",
+      "query": {
+        "sparse_vector": {
+          "field": "inference_field.inference.chunks.embeddings",
+          "inference_id": "my-inference-id",
+          "query": "mountain lake"
+        }
+      }
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+You can customize the `sparse_vector` query to include specific settings, like <<sparse-vector-query-with-pruning-config-and-rescore-example,pruning configuration>>.
+
+
+[discrete]
+[[search-text-inferece]]
+==== Search with `text_embedding` inference
+
+When the {infer} endpoint uses a `text_embedding` model, you can use a <<query-dsl-knn-query,`knn` query>> on a `semantic_text` field in the following way:
+
+[source,console]
+------------------------------------------------------------
+GET test-index/_search
+{
+  "query": {
+    "nested": {
+      "path": "inference_field.inference.chunks",
+      "query": {
+        "knn": {
+          "field": "inference_field.inference.chunks.embeddings",
+          "query_vector_builder": {
+            "text_embedding": {
+              "model_id": "my_inference_id",
+	      "model_text": "mountain lake"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+You can customize the `knn` query to include specific settings, like `num_candidates` and `k`.
diff --git a/docs/reference/query-dsl/special-queries.asciidoc b/docs/reference/query-dsl/special-queries.asciidoc
index 8fb23ca4dbb64..90cd9a696a6d9 100644
--- a/docs/reference/query-dsl/special-queries.asciidoc
+++ b/docs/reference/query-dsl/special-queries.asciidoc
@@ -32,6 +32,9 @@ This query allows a script to act as a filter. Also see the
 <<query-dsl-script-score-query,`script_score` query>>::
 A query that allows to modify the score of a sub-query with a script.
 
+<<query-dsl-semantic-query,`semantic` query>>::
+A query that allows you to perform semantic search.
+
 <<query-dsl-wrapper-query,`wrapper` query>>::
 A query that accepts other queries as json or yaml string.
 
@@ -55,6 +58,8 @@ include::script-query.asciidoc[]
 
 include::script-score-query.asciidoc[]
 
+include::semantic-query.asciidoc[]
+
 include::wrapper-query.asciidoc[]
 
 include::pinned-query.asciidoc[]
diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc
new file mode 100644
index 0000000000000..c2dabedb0336c
--- /dev/null
+++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc
@@ -0,0 +1,267 @@
+[[semantic-search-semantic-text]]
+=== Tutorial: semantic search with `semantic_text`
+++++
+<titleabbrev>Semantic search with `semantic_text`</titleabbrev>
+++++
+
+beta[]
+
+This tutorial shows you how to use the semantic text feature to perform semantic search on your data.
+
+Semantic text simplifies the {infer} workflow by providing {infer} at ingestion time and sensible default values automatically.
+You don't need to define model related settings and parameters, or create {infer} ingest pipelines.
+
+The recommended way to use <<semantic-search,semantic search>> in the {stack} is following the `semantic_text` workflow.
+When you need more control over indexing and query settings, you can still use the complete {infer} workflow (refer to  <<semantic-search-inference,this tutorial>> to review the process).
+
+This tutorial uses the <<inference-example-elser,`elser` service>> for demonstration, but you can use any service and their supported models offered by the {infer-cap} API.
+
+
+[discrete]
+[[semantic-text-requirements]]
+==== Requirements
+
+To use the `semantic_text` field type, you must have an {infer} endpoint deployed in
+your cluster using the <<put-inference-api>>.
+
+
+[discrete]
+[[semantic-text-infer-endpoint]]
+==== Create the {infer} endpoint
+
+Create an inference endpoint by using the <<put-inference-api>>:
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/sparse_embedding/my-elser-endpoint <1>
+{
+  "service": "elser", <2>
+  "service_settings": {
+    "num_allocations": 1,
+    "num_threads": 1
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The task type is `sparse_embedding` in the path as the `elser` service will
+be used and ELSER creates sparse vectors. The `inference_id` is
+`my-elser-endpoint`.
+<2> The `elser` service is used in this example.
+
+
+[discrete]
+[[semantic-text-index-mapping]]
+==== Create the index mapping
+
+The mapping of the destination index - the index that contains the embeddings
+that the inference endpoint will generate based on your input text - must be created. The
+destination index must have a field with the <<semantic-text,`semantic_text`>>
+field type to index the output of the used inference endpoint.
+
+[source,console]
+------------------------------------------------------------
+PUT semantic-embeddings
+{
+  "mappings": {
+    "properties": {
+      "semantic_text": { <1>
+        "type": "semantic_text", <2>
+        "inference_id": "my-elser-endpoint" <3>
+      },
+      "content": { <4>
+        "type": "text",
+        "copy_to": "semantic_text" <5>
+      }
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The name of the field to contain the generated embeddings.
+<2> The field to contain the embeddings is a `semantic_text` field.
+<3> The `inference_id` is the inference endpoint you created in the previous step.
+It will be used to generate the embeddings based on the input text.
+Every time you ingest data into the related `semantic_text` field, this endpoint will be used for creating the vector representation of the text.
+<4> The field to store the text reindexed from a source index in the  <<semantic-text-reindex-data,Reindex the data>> step.
+<5> The textual data stored in the `content` field will be copied to `semantic_text` and processed by the {infer} endpoint.
+The `semantic_text` field will store the embeddings generated based on the input data.
+
+
+[discrete]
+[[semantic-text-load-data]]
+==== Load data
+
+In this step, you load the data that you later use to create embeddings from it.
+
+Use the `msmarco-passagetest2019-top1000` data set, which is a subset of the MS
+MARCO Passage Ranking data set. It consists of 200 queries, each accompanied by
+a list of relevant text passages. All unique passages, along with their IDs,
+have been extracted from that data set and compiled into a
+https://github.com/elastic/stack-docs/blob/main/docs/en/stack/ml/nlp/data/msmarco-passagetest2019-unique.tsv[tsv file].
+
+Download the file and upload it to your cluster using the
+{kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer]
+in the {ml-app} UI. Assign the name `id` to the first column and `content` to
+the second column. The index name is `test-data`. Once the upload is complete,
+you can see an index named `test-data` with 182469 documents.
+
+
+[discrete]
+[[semantic-text-reindex-data]]
+==== Reindex the data
+
+Create the embeddings from the text by reindexing the data from the `test-data`
+index to the `semantic-embeddings` index. The data in the `content` field will
+be reindexed into the `content` field of the destination index. 
+The `content` field data will be copied to the `semantic_text` field as a result of the `copy_to`
+parameter set in the index mapping creation step. The copied data will be
+processed by the {infer} endpoint associated with the `semantic_text` semantic text
+field.
+
+[source,console]
+------------------------------------------------------------
+POST _reindex?wait_for_completion=false
+{
+  "source": { 
+    "index": "test-data",
+    "size": 10 <1>
+  },
+  "dest": {
+    "index": "semantic-embeddings"
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The default batch size for reindexing is 1000. Reducing size to a smaller
+number makes the update of the reindexing process quicker which enables you to
+follow the progress closely and detect errors early.
+
+The call returns a task ID to monitor the progress:
+
+[source,console]
+------------------------------------------------------------
+GET _tasks/<task_id>
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+It is recommended to cancel the reindexing process if you don't want to wait
+until it is fully complete which might take a long time for an inference endpoint with few assigned resources:
+
+[source,console]
+------------------------------------------------------------
+POST _tasks/<task_id>/_cancel
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+
+[discrete]
+[[semantic-text-semantic-search]]
+==== Semantic search
+
+After the data set has been enriched with the embeddings, you can query the data
+using semantic search. Provide the `semantic_text` field name and the query text
+in a `semantic` query type. The {infer} endpoint used to generate the embeddings
+for the `semantic_text` field will be used to process the query text.
+
+[source,console]
+------------------------------------------------------------
+GET semantic-embeddings/_search
+{
+  "query": {
+    "semantic": { 
+      "field": "semantic_text", <1>
+      "query": "How to avoid muscle soreness while running?" <2>
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The `semantic_text` field on which you want to perform the search.
+<2> The query text.
+
+As a result, you receive the top 10 documents that are closest in meaning to the
+query from the `semantic-embedding` index:
+
+[source,console-result]
+------------------------------------------------------------
+"hits": [
+  {
+    "_index": "semantic-embeddings",
+    "_id": "6DdEuo8B0vYIvzmhoEtt",
+    "_score": 24.972616,
+    "_source": {
+      "semantic_text": {
+        "inference": {
+          "inference_id": "my-elser-endpoint",
+          "model_settings": {
+            "task_type": "sparse_embedding"
+          },
+          "chunks": [
+            {
+              "text": "There are a few foods and food groups that will help to fight inflammation and delayed onset muscle soreness (both things that are inevitable after a long, hard workout) when you incorporate them into your postworkout eats, whether immediately after your run or at a meal later in the day. Advertisement. Advertisement.",
+              "embeddings": {
+                (...)
+              }
+            }
+          ]
+        }
+      },
+      "id": 1713868,
+      "content": "There are a few foods and food groups that will help to fight inflammation and delayed onset muscle soreness (both things that are inevitable after a long, hard workout) when you incorporate them into your postworkout eats, whether immediately after your run or at a meal later in the day. Advertisement. Advertisement."
+    }
+  },
+  {
+    "_index": "semantic-embeddings",
+    "_id": "-zdEuo8B0vYIvzmhplLX",
+    "_score": 22.143118,
+    "_source": {
+      "semantic_text": {
+        "inference": {
+          "inference_id": "my-elser-endpoint",
+          "model_settings": {
+            "task_type": "sparse_embedding"
+          },
+          "chunks": [
+            {
+              "text": "During Your Workout. There are a few things you can do during your workout to help prevent muscle injury and soreness. According to personal trainer and writer for Iron Magazine, Marc David, doing warm-ups and cool-downs between sets can help keep muscle soreness to a minimum.",
+              "embeddings": {
+                (...)
+              }
+            }
+          ]
+        }
+      },
+      "id": 3389244,
+      "content": "During Your Workout. There are a few things you can do during your workout to help prevent muscle injury and soreness. According to personal trainer and writer for Iron Magazine, Marc David, doing warm-ups and cool-downs between sets can help keep muscle soreness to a minimum."
+    }
+  },
+  {
+    "_index": "semantic-embeddings",
+    "_id": "77JEuo8BdmhTuQdXtQWt",
+    "_score": 21.506052,
+    "_source": {
+      "semantic_text": {
+        "inference": {
+          "inference_id": "my-elser-endpoint",
+          "model_settings": {
+            "task_type": "sparse_embedding"
+          },
+          "chunks": [
+            {
+              "text": "This is especially important if the soreness is due to a weightlifting routine. For this time period, do not exert more than around 50% of the level of effort (weight, distance and speed) that caused the muscle groups to be sore.",
+              "embeddings": {
+                (...)
+              }
+            }
+          ]
+        }
+      },
+      "id": 363742,
+      "content": "This is especially important if the soreness is due to a weightlifting routine. For this time period, do not exert more than around 50% of the level of effort (weight, distance and speed) that caused the muscle groups to be sore."
+    }
+  },
+  (...)
+]
+------------------------------------------------------------
+// NOTCONSOLE
+
diff --git a/docs/reference/search/search-your-data/semantic-search.asciidoc b/docs/reference/search/search-your-data/semantic-search.asciidoc
index a1197e7bbbd3a..7dd7fbaf9823c 100644
--- a/docs/reference/search/search-your-data/semantic-search.asciidoc
+++ b/docs/reference/search/search-your-data/semantic-search.asciidoc
@@ -135,5 +135,6 @@ include::{es-ref-dir}/tab-widgets/semantic-search/hybrid-search-widget.asciidoc[
 ** The https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs`] repo contains a number of interactive semantic search examples in the form of executable Python notebooks, using the {es} Python client
 
 include::semantic-search-elser.asciidoc[]
+include::semantic-search-semantic-text.asciidoc[]
 include::semantic-search-inference.asciidoc[]
 include::cohere-es.asciidoc[]

From c93862f1bdf7935cf658ba0afcd5992d913f8aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= <aurelien.foucret@gmail.com>
Date: Fri, 31 May 2024 18:02:54 +0200
Subject: [PATCH 085/208] Wrap "Pattern too complex" exception into an
 IllegalArgumentException (#109173)

---
 docs/changelog/109173.yaml                    |  5 ++++
 .../test/analysis-common/50_char_filters.yml  | 27 +++++++++++++++++++
 .../org/elasticsearch/common/regex/Regex.java | 22 +++++++++++++--
 3 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100644 docs/changelog/109173.yaml

diff --git a/docs/changelog/109173.yaml b/docs/changelog/109173.yaml
new file mode 100644
index 0000000000000..9f4f73a6f74c8
--- /dev/null
+++ b/docs/changelog/109173.yaml
@@ -0,0 +1,5 @@
+pr: 109173
+summary: Wrap "Pattern too complex" exception into an `IllegalArgumentException`
+area: Mapping
+type: bug
+issues: []
diff --git a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml
index 67e68428c07c7..76f17dddd3f0e 100644
--- a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml
+++ b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml
@@ -27,6 +27,33 @@
     - length: { tokens: 1 }
     - match:  { tokens.0.token: "replacedSample 6 sample1" }
 
+---
+"pattern_replace error handling (too complex pattern)":
+  - do:
+      catch: bad_request
+      indices.create:
+        index: test_too_complex_regex_pattern
+        body:
+          settings:
+            index:
+              analysis:
+                analyzer:
+                  my_analyzer:
+                    tokenizer: standard
+                    char_filter:
+                      - my_char_filter
+                char_filter:
+                  my_char_filter:
+                    type: "pattern_replace"
+                    # This pattern intentionally uses special characters designed to throw an error.
+                    # It's expected that the pattern may not render correctly.
+                    pattern: "(\\d+)-(?=\\d\nͭͭͭͭͭͭͭͭͭͭͭͭͭͭͭ"
+                    flags: CASE_INSENSITIVE|MULTILINE|DOTALL|UNICODE_CASE|CANON_EQ
+                    replacement: "_$1"
+  - match: { status: 400 }
+  - match: { error.type: illegal_argument_exception }
+  - match: { error.reason: "Too complex regex pattern" }
+
 ---
 "mapping":
     - do:
diff --git a/server/src/main/java/org/elasticsearch/common/regex/Regex.java b/server/src/main/java/org/elasticsearch/common/regex/Regex.java
index 039f484f1ebca..983144c7cee89 100644
--- a/server/src/main/java/org/elasticsearch/common/regex/Regex.java
+++ b/server/src/main/java/org/elasticsearch/common/regex/Regex.java
@@ -230,8 +230,26 @@ public static boolean simpleMatch(final List<String> patterns, final String str)
     }
 
     public static Pattern compile(String regex, String flags) {
-        int pFlags = flags == null ? 0 : flagsFromString(flags);
-        return Pattern.compile(regex, pFlags);
+        try {
+            int pFlags = flags == null ? 0 : flagsFromString(flags);
+            return Pattern.compile(regex, pFlags);
+        } catch (OutOfMemoryError e) {
+            if (e.getMessage().equals("Pattern too complex")) {
+                // Normally, we do try to handle OutOfMemoryError errors, as they typically indicate the JVM is not healthy.
+                //
+                // In the context of Pattern::compile, an OutOfMemoryError can occur if the pattern is too complex.
+                // In this case, the OutOfMemoryError is thrown by a pre-check rather than actual memory exhaustion.
+                //
+                // Because the JVM has not encountered a real memory issue, we can treat this as a recoverable exception by wrapping
+                // the original OutOfMemoryError in an IllegalArgumentException.
+                //
+                // For additional details, see:
+                // - https://bugs.openjdk.org/browse/JDK-8300207
+                // - https://github.com/openjdk/jdk/commit/030b071db1fb6197a2633a04b20aa95432a903bc
+                throw new IllegalArgumentException("Too complex regex pattern", e);
+            }
+            throw e;
+        }
     }
 
     public static int flagsFromString(String flags) {

From 0339d8f386b51ee9a7ebe148e15835747f78c12f Mon Sep 17 00:00:00 2001
From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com>
Date: Fri, 31 May 2024 13:13:54 -0400
Subject: [PATCH 086/208] Wait for executor in start (#109253)

---
 .../http/sender/HttpRequestSender.java        | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java
index d337860848160..21a758a3db248 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java
@@ -73,7 +73,7 @@ public Sender createSender(String serviceName) {
     private final HttpClientManager manager;
     private final RequestExecutorService service;
     private final AtomicBoolean started = new AtomicBoolean(false);
-    private final CountDownLatch startCompleted = new CountDownLatch(2);
+    private final CountDownLatch startCompleted = new CountDownLatch(1);
 
     private HttpRequestSender(
         String serviceName,
@@ -103,7 +103,17 @@ public void start() {
             // is ready prior to the service attempting to use the http client to send a request
             manager.start();
             threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(service::start);
-            startCompleted.countDown();
+            waitForStartToComplete();
+        }
+    }
+
+    private void waitForStartToComplete() {
+        try {
+            if (startCompleted.await(START_COMPLETED_WAIT_TIME.getSeconds(), TimeUnit.SECONDS) == false) {
+                throw new IllegalStateException("Http sender startup did not complete in time");
+            }
+        } catch (InterruptedException e) {
+            throw new IllegalStateException("Http sender interrupted while waiting for startup to complete");
         }
     }
 
@@ -133,14 +143,4 @@ public void send(
         waitForStartToComplete();
         service.execute(requestCreator, inferenceInputs, timeout, listener);
     }
-
-    private void waitForStartToComplete() {
-        try {
-            if (startCompleted.await(START_COMPLETED_WAIT_TIME.getSeconds(), TimeUnit.SECONDS) == false) {
-                throw new IllegalStateException("Http sender startup did not complete in time");
-            }
-        } catch (InterruptedException e) {
-            throw new IllegalStateException("Http sender interrupted while waiting for startup to complete");
-        }
-    }
 }

From 2a0f2e360d5b03b7d58363adcb10102af3c0501d Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Fri, 31 May 2024 13:39:55 -0400
Subject: [PATCH 087/208] Mute
 org.elasticsearch.blobcache.common.SparseFileTrackerTests
 testCallsListenerWhenRangeIsAvailable #109259

---
 muted-tests.yml | 94 +++++++++++++++++++++++++------------------------
 1 file changed, 48 insertions(+), 46 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 40907992f58a3..b3cd78c902a78 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -1,50 +1,52 @@
 tests:
-- class: "org.elasticsearch.xpack.transform.transforms.scheduling.MonotonicClockTests"
-  issue: "https://github.com/elastic/elasticsearch/issues/108529"
-- class: "org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterTests"
-  issue: "https://github.com/elastic/elasticsearch/issues/108649"
-  method: "testManyRandomDocs"
-- class: "org.elasticsearch.cluster.coordination.CoordinatorVotingConfigurationTests"
-  issue: "https://github.com/elastic/elasticsearch/issues/108729"
-  method: "testClusterUUIDLogging"
-- class: "org.elasticsearch.xpack.core.ssl.SSLConfigurationReloaderTests"
-  issue: "https://github.com/elastic/elasticsearch/issues/108774"
-  method: "testReloadingKeyStore"
-- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108808"
-  method: "test {k8s-metrics.MetricsWithAggs}"
-- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108809"
-  method: "test {k8s-metrics.MetricsWithoutAggs}"
-- class: "org.elasticsearch.xpack.textstructure.structurefinder.TimestampFormatFinderTests"
-  issue: "https://github.com/elastic/elasticsearch/issues/108855"
-  method: "testGuessIsDayFirstFromLocale"
-- class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108857"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
-    \ dependent mappings / dates}"
-- class: "org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108950"
-  method: "test {p0=health/10_basic/cluster health basic test}"
-- class: "org.elasticsearch.upgrades.SearchStatesIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108991"
-  method: "testCanMatch"
-- class: "org.elasticsearch.upgrades.AggregationsIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108992"
-  method: "testHistogram"
-- class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108993"
-  method: "testTrainedModelInference"
-- class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109188"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
-    \ dependent mappings / dates}"
-- class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109189"
-  method: "test {p0=esql/70_locale/Date format with Italian locale}"
-- class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109200"
-  method: "test {p0=esql/70_locale/Date format with Italian locale}"
+  - class: "org.elasticsearch.xpack.transform.transforms.scheduling.MonotonicClockTests"
+    issue: "https://github.com/elastic/elasticsearch/issues/108529"
+  - class: "org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterTests"
+    issue: "https://github.com/elastic/elasticsearch/issues/108649"
+    method: "testManyRandomDocs"
+  - class: "org.elasticsearch.cluster.coordination.CoordinatorVotingConfigurationTests"
+    issue: "https://github.com/elastic/elasticsearch/issues/108729"
+    method: "testClusterUUIDLogging"
+  - class: "org.elasticsearch.xpack.core.ssl.SSLConfigurationReloaderTests"
+    issue: "https://github.com/elastic/elasticsearch/issues/108774"
+    method: "testReloadingKeyStore"
+  - class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
+    issue: "https://github.com/elastic/elasticsearch/issues/108808"
+    method: "test {k8s-metrics.MetricsWithAggs}"
+  - class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
+    issue: "https://github.com/elastic/elasticsearch/issues/108809"
+    method: "test {k8s-metrics.MetricsWithoutAggs}"
+  - class: "org.elasticsearch.xpack.textstructure.structurefinder.TimestampFormatFinderTests"
+    issue: "https://github.com/elastic/elasticsearch/issues/108855"
+    method: "testGuessIsDayFirstFromLocale"
+  - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
+    issue: "https://github.com/elastic/elasticsearch/issues/108857"
+    method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+  - class: "org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT"
+    issue: "https://github.com/elastic/elasticsearch/issues/108950"
+    method: "test {p0=health/10_basic/cluster health basic test}"
+  - class: "org.elasticsearch.upgrades.SearchStatesIT"
+    issue: "https://github.com/elastic/elasticsearch/issues/108991"
+    method: "testCanMatch"
+  - class: "org.elasticsearch.upgrades.AggregationsIT"
+    issue: "https://github.com/elastic/elasticsearch/issues/108992"
+    method: "testHistogram"
+  - class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT"
+    issue: "https://github.com/elastic/elasticsearch/issues/108993"
+    method: "testTrainedModelInference"
+  - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
+    issue: "https://github.com/elastic/elasticsearch/issues/109188"
+    method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+  - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
+    issue: "https://github.com/elastic/elasticsearch/issues/109189"
+    method: "test {p0=esql/70_locale/Date format with Italian locale}"
+  - class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
+    issue: "https://github.com/elastic/elasticsearch/issues/109200"
+    method: "test {p0=esql/70_locale/Date format with Italian locale}"
+  - class: org.elasticsearch.blobcache.common.SparseFileTrackerTests
+    method: testCallsListenerWhenRangeIsAvailable
+    issue: https://github.com/elastic/elasticsearch/issues/109259
+
 # Examples:
 #
 #  Mute a single test case in a YAML test suite:

From 8295e1885dc510f7e2ce2cbe6778913636130a2b Mon Sep 17 00:00:00 2001
From: Pat Whelan <pat.whelan@elastic.co>
Date: Fri, 31 May 2024 13:43:05 -0400
Subject: [PATCH 088/208] [ML] Refactor InferenceRunner (#109228)

A future change will refactor how InferenceRunner's `run` method
functions, and I want to reuse the existing unit tests to verify the
behavior does not change.

Relate #109134
---
 .../dataframe/DataFrameAnalyticsManager.java  |   2 +-
 .../dataframe/inference/InferenceRunner.java  |  57 ++++---
 .../dataframe/inference/TestDocsIterator.java |   2 +-
 .../inference/InferenceRunnerTests.java       | 148 ++++++++++++------
 4 files changed, 143 insertions(+), 66 deletions(-)

diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java
index 203474a3c9d0a..13f13a271c452 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java
@@ -297,7 +297,7 @@ private void buildInferenceStep(DataFrameAnalyticsTask task, DataFrameAnalyticsC
             config,
             listener.delegateFailureAndWrap((delegate, extractedFieldsDetector) -> {
                 ExtractedFields extractedFields = extractedFieldsDetector.detect().v1();
-                InferenceRunner inferenceRunner = new InferenceRunner(
+                InferenceRunner inferenceRunner = InferenceRunner.create(
                     settings,
                     parentTaskClient,
                     modelLoadingService,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java
index 06075363997c7..073fb13cbf420 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java
@@ -49,6 +49,7 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
 
 import static org.elasticsearch.core.Strings.format;
 
@@ -67,9 +68,10 @@ public class InferenceRunner {
     private final ExtractedFields extractedFields;
     private final ProgressTracker progressTracker;
     private final DataCountsTracker dataCountsTracker;
+    private final Function<Long, TestDocsIterator> testDocsIteratorFactory;
     private volatile boolean isCancelled;
 
-    public InferenceRunner(
+    InferenceRunner(
         Settings settings,
         Client client,
         ModelLoadingService modelLoadingService,
@@ -78,7 +80,8 @@ public InferenceRunner(
         DataFrameAnalyticsConfig config,
         ExtractedFields extractedFields,
         ProgressTracker progressTracker,
-        DataCountsTracker dataCountsTracker
+        DataCountsTracker dataCountsTracker,
+        Function<Long, TestDocsIterator> testDocsIteratorFactory
     ) {
         this.settings = Objects.requireNonNull(settings);
         this.client = Objects.requireNonNull(client);
@@ -89,6 +92,7 @@ public InferenceRunner(
         this.extractedFields = Objects.requireNonNull(extractedFields);
         this.progressTracker = Objects.requireNonNull(progressTracker);
         this.dataCountsTracker = Objects.requireNonNull(dataCountsTracker);
+        this.testDocsIteratorFactory = Objects.requireNonNull(testDocsIteratorFactory);
     }
 
     public void cancel() {
@@ -108,12 +112,7 @@ public void run(String modelId) {
             modelLoadingService.getModelForInternalInference(modelId, localModelPlainActionFuture);
             InferenceState inferenceState = restoreInferenceState();
             dataCountsTracker.setTestDocsCount(inferenceState.processedTestDocsCount);
-            TestDocsIterator testDocsIterator = new TestDocsIterator(
-                new OriginSettingClient(client, ClientHelper.ML_ORIGIN),
-                config,
-                extractedFields,
-                inferenceState.lastIncrementalId
-            );
+            TestDocsIterator testDocsIterator = testDocsIteratorFactory.apply(inferenceState.lastIncrementalId);
             try (LocalModel localModel = localModelPlainActionFuture.actionGet()) {
                 LOGGER.debug("Loaded inference model [{}]", localModel);
                 inferTestDocs(localModel, testDocsIterator, inferenceState.processedTestDocsCount);
@@ -179,8 +178,7 @@ private InferenceState restoreInferenceState() {
         }
     }
 
-    // Visible for testing
-    void inferTestDocs(LocalModel model, TestDocsIterator testDocsIterator, long processedTestDocsCount) {
+    private void inferTestDocs(LocalModel model, TestDocsIterator testDocsIterator, long processedTestDocsCount) {
         long totalDocCount = 0;
         long processedDocCount = processedTestDocsCount;
 
@@ -248,14 +246,35 @@ private void executeBulkRequest(BulkRequest bulkRequest) {
         );
     }
 
-    private static class InferenceState {
-
-        private final Long lastIncrementalId;
-        private final long processedTestDocsCount;
-
-        InferenceState(@Nullable Long lastIncrementalId, long processedTestDocsCount) {
-            this.lastIncrementalId = lastIncrementalId;
-            this.processedTestDocsCount = processedTestDocsCount;
-        }
+    public static InferenceRunner create(
+        Settings settings,
+        Client client,
+        ModelLoadingService modelLoadingService,
+        ResultsPersisterService resultsPersisterService,
+        TaskId parentTaskId,
+        DataFrameAnalyticsConfig config,
+        ExtractedFields extractedFields,
+        ProgressTracker progressTracker,
+        DataCountsTracker dataCountsTracker
+    ) {
+        return new InferenceRunner(
+            settings,
+            client,
+            modelLoadingService,
+            resultsPersisterService,
+            parentTaskId,
+            config,
+            extractedFields,
+            progressTracker,
+            dataCountsTracker,
+            lastIncrementalId -> new TestDocsIterator(
+                new OriginSettingClient(client, ClientHelper.ML_ORIGIN),
+                config,
+                extractedFields,
+                lastIncrementalId
+            )
+        );
     }
+
+    private record InferenceState(@Nullable Long lastIncrementalId, long processedTestDocsCount) {}
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/TestDocsIterator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/TestDocsIterator.java
index 9e2db58befdbf..b5dfe14e0834b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/TestDocsIterator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/TestDocsIterator.java
@@ -27,7 +27,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class TestDocsIterator extends SearchAfterDocumentsIterator<SearchHit> {
+class TestDocsIterator extends SearchAfterDocumentsIterator<SearchHit> {
 
     private final DataFrameAnalyticsConfig config;
     private Long lastDocId;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java
index 78ee3e1d6e4fa..ad6b68e1051ff 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java
@@ -7,18 +7,32 @@
 
 package org.elasticsearch.xpack.ml.dataframe.inference;
 
+import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.inference.InferenceResults;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchResponseUtils;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.metrics.Max;
+import org.elasticsearch.search.profile.SearchProfileResults;
+import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
@@ -28,7 +42,7 @@
 import org.elasticsearch.xpack.core.ml.dataframe.stats.common.DataCounts;
 import org.elasticsearch.xpack.core.ml.inference.results.ClassificationInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfig;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig;
+import org.elasticsearch.xpack.ml.dataframe.DestinationIndex;
 import org.elasticsearch.xpack.ml.dataframe.stats.DataCountsTracker;
 import org.elasticsearch.xpack.ml.dataframe.stats.ProgressTracker;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
@@ -42,16 +56,18 @@
 
 import java.io.IOException;
 import java.util.ArrayDeque;
-import java.util.Arrays;
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.Deque;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -67,7 +83,7 @@ public class InferenceRunnerTests extends ESTestCase {
 
     @Before
     public void setupTests() {
-        client = mock(Client.class);
+        client = mockClient();
         resultsPersisterService = mock(ResultsPersisterService.class);
         config = new DataFrameAnalyticsConfig.Builder().setId("test")
             .setAnalysis(RegressionTests.createRandom())
@@ -80,32 +96,28 @@ public void setupTests() {
     }
 
     public void testInferTestDocs() {
-        ExtractedFields extractedFields = new ExtractedFields(
-            Collections.singletonList(new SourceField("key", Collections.singleton("integer"))),
-            Collections.emptyList(),
-            Collections.emptyMap()
-        );
+        var extractedFields = new ExtractedFields(List.of(new SourceField("key", Set.of("integer"))), List.of(), Map.of());
 
-        Map<String, Object> doc1 = new HashMap<>();
-        doc1.put("key", 1);
-        Map<String, Object> doc2 = new HashMap<>();
-        doc2.put("key", 2);
-        TestDocsIterator testDocsIterator = mock(TestDocsIterator.class);
+        var testDocsIterator = mock(TestDocsIterator.class);
         when(testDocsIterator.hasNext()).thenReturn(true, false);
-        when(testDocsIterator.next()).thenReturn(buildSearchHits(Arrays.asList(doc1, doc2)));
+        when(testDocsIterator.next()).thenReturn(buildSearchHits(List.of(Map.of("key", 1), Map.of("key", 2))));
         when(testDocsIterator.getTotalHits()).thenReturn(2L);
-        InferenceConfig config = ClassificationConfig.EMPTY_PARAMS;
+        var config = ClassificationConfig.EMPTY_PARAMS;
 
-        LocalModel localModel = localModelInferences(
-            new ClassificationInferenceResults(1.0, "foo", Collections.emptyList(), Collections.emptyList(), config, 1.0, 1.0),
-            new ClassificationInferenceResults(0.0, "bar", Collections.emptyList(), Collections.emptyList(), config, .5, .7)
+        var localModel = localModelInferences(
+            new ClassificationInferenceResults(1.0, "foo", List.of(), List.of(), config, 1.0, 1.0),
+            new ClassificationInferenceResults(0.0, "bar", List.of(), List.of(), config, .5, .7)
         );
 
-        InferenceRunner inferenceRunner = createInferenceRunner(extractedFields);
+        doAnswer(ans -> {
+            ActionListener<LocalModel> responseListener = ans.getArgument(1);
+            responseListener.onResponse(localModel);
+            return null;
+        }).when(modelLoadingService).getModelForInternalInference(anyString(), any());
 
-        inferenceRunner.inferTestDocs(localModel, testDocsIterator, 0L);
+        createInferenceRunner(extractedFields, testDocsIterator).run("model id");
 
-        ArgumentCaptor<BulkRequest> argumentCaptor = ArgumentCaptor.forClass(BulkRequest.class);
+        var argumentCaptor = ArgumentCaptor.forClass(BulkRequest.class);
 
         verify(resultsPersisterService).bulkIndexWithHeadersWithRetry(any(), argumentCaptor.capture(), any(), any(), any());
         assertThat(progressTracker.getInferenceProgressPercent(), equalTo(100));
@@ -115,23 +127,14 @@ public void testInferTestDocs() {
         Map<String, Object> doc1Source = ((IndexRequest) indexRequests.get(0)).sourceAsMap();
         Map<String, Object> doc2Source = ((IndexRequest) indexRequests.get(1)).sourceAsMap();
 
-        assertThat(doc1Source.get("test_results_field"), equalTo(new HashMap<>() {
-            {
-                put("predicted_value", "foo");
-                put("prediction_probability", 1.0);
-                put("prediction_score", 1.0);
-                put("predicted_value", "foo");
-                put("is_training", false);
-            }
-        }));
-        assertThat(doc2Source.get("test_results_field"), equalTo(new HashMap<>() {
-            {
-                put("predicted_value", "bar");
-                put("prediction_probability", 0.5);
-                put("prediction_score", .7);
-                put("is_training", false);
-            }
-        }));
+        assertThat(
+            doc1Source.get("test_results_field"),
+            equalTo(Map.of("predicted_value", "foo", "prediction_probability", 1.0, "prediction_score", 1.0, "is_training", false))
+        );
+        assertThat(
+            doc2Source.get("test_results_field"),
+            equalTo(Map.of("predicted_value", "bar", "prediction_probability", 0.5, "prediction_score", .7, "is_training", false))
+        );
     }
 
     public void testInferTestDocs_GivenCancelWasCalled() {
@@ -141,10 +144,10 @@ public void testInferTestDocs_GivenCancelWasCalled() {
         TestDocsIterator infiniteDocsIterator = mock(TestDocsIterator.class);
         when(infiniteDocsIterator.hasNext()).thenReturn(true);
 
-        InferenceRunner inferenceRunner = createInferenceRunner(extractedFields);
+        InferenceRunner inferenceRunner = createInferenceRunner(extractedFields, infiniteDocsIterator);
         inferenceRunner.cancel();
 
-        inferenceRunner.inferTestDocs(localModel, infiniteDocsIterator, 0L);
+        inferenceRunner.run("model id");
 
         Mockito.verifyNoMoreInteractions(localModel, resultsPersisterService);
         assertThat(progressTracker.getInferenceProgressPercent(), equalTo(0));
@@ -152,7 +155,7 @@ public void testInferTestDocs_GivenCancelWasCalled() {
 
     private static Deque<SearchHit> buildSearchHits(List<Map<String, Object>> vals) {
         return vals.stream().map(InferenceRunnerTests::fromMap).map(reference -> {
-            var pooled = SearchResponseUtils.searchHitFromMap(Collections.singletonMap("_source", reference));
+            var pooled = SearchResponseUtils.searchHitFromMap(Map.of("_source", reference));
             try {
                 return pooled.asUnpooled();
             } finally {
@@ -175,7 +178,7 @@ private LocalModel localModelInferences(InferenceResults first, InferenceResults
         return localModel;
     }
 
-    private InferenceRunner createInferenceRunner(ExtractedFields extractedFields) {
+    private InferenceRunner createInferenceRunner(ExtractedFields extractedFields, TestDocsIterator testDocsIterator) {
         return new InferenceRunner(
             Settings.EMPTY,
             client,
@@ -185,7 +188,62 @@ private InferenceRunner createInferenceRunner(ExtractedFields extractedFields) {
             config,
             extractedFields,
             progressTracker,
-            new DataCountsTracker(new DataCounts(config.getId()))
+            new DataCountsTracker(new DataCounts(config.getId())),
+            id -> testDocsIterator
         );
     }
+
+    private Client mockClient() {
+        var client = mock(Client.class);
+        var threadpool = mock(ThreadPool.class);
+        when(threadpool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
+        when(client.threadPool()).thenReturn(threadpool);
+
+        Supplier<SearchResponse> withHits = () -> new SearchResponse(
+            SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f),
+            InternalAggregations.from(List.of(new Max(DestinationIndex.INCREMENTAL_ID, 1, DocValueFormat.RAW, Map.of()))),
+            new Suggest(new ArrayList<>()),
+            false,
+            false,
+            new SearchProfileResults(Map.of()),
+            1,
+            "",
+            1,
+            1,
+            0,
+            0,
+            ShardSearchFailure.EMPTY_ARRAY,
+            SearchResponse.Clusters.EMPTY
+        );
+        Supplier<SearchResponse> withNoHits = () -> new SearchResponse(
+            SearchHits.EMPTY_WITH_TOTAL_HITS,
+            // Simulate completely null aggs
+            null,
+            new Suggest(new ArrayList<>()),
+            false,
+            false,
+            new SearchProfileResults(Map.of()),
+            1,
+            "",
+            1,
+            1,
+            0,
+            0,
+            ShardSearchFailure.EMPTY_ARRAY,
+            SearchResponse.Clusters.EMPTY
+        );
+
+        when(client.search(any())).thenReturn(response(withHits)).thenReturn(response(withNoHits));
+        return client;
+    }
+
+    // we only expect to call actionGet, calling other API will hang indefinitely
+    private static ActionFuture<SearchResponse> response(Supplier<SearchResponse> searchResponse) {
+        return new PlainActionFuture<>() {
+            @Override
+            public SearchResponse actionGet() {
+                return searchResponse.get();
+            }
+        };
+    }
 }

From c84634444af0b5a12dee26ce25d6abfe3198ce80 Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Fri, 31 May 2024 13:48:41 -0400
Subject: [PATCH 089/208] Mute
 org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT test
 {yaml=reference/esql/esql-async-query-api/line_17} #109260

---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index b3cd78c902a78..cc7c0be9bf9fb 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -46,6 +46,9 @@ tests:
   - class: org.elasticsearch.blobcache.common.SparseFileTrackerTests
     method: testCallsListenerWhenRangeIsAvailable
     issue: https://github.com/elastic/elasticsearch/issues/109259
+  - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+    method: test {yaml=reference/esql/esql-async-query-api/line_17}
+    issue: https://github.com/elastic/elasticsearch/issues/109260
 
 # Examples:
 #

From 1c4b59497aae0da0458958542f6642fb0858e2f6 Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Fri, 31 May 2024 13:57:21 -0400
Subject: [PATCH 090/208] Mute
 org.elasticsearch.xpack.ccr.LocalIndexFollowingIT testRemoveRemoteConnection
 #109163

---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index cc7c0be9bf9fb..c7a18b527e8ce 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -49,6 +49,9 @@ tests:
   - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
     method: test {yaml=reference/esql/esql-async-query-api/line_17}
     issue: https://github.com/elastic/elasticsearch/issues/109260
+  - class: org.elasticsearch.xpack.ccr.LocalIndexFollowingIT
+    method: testRemoveRemoteConnection
+    issue: https://github.com/elastic/elasticsearch/issues/109163
 
 # Examples:
 #

From f7ce9596a8de8bade38df94d16ee0f39af1c3cf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= <aurelien.foucret@gmail.com>
Date: Fri, 31 May 2024 20:14:58 +0200
Subject: [PATCH 091/208] Better handling of multiple rescorers clauses with
 LTR. (#109071)

---
 docs/changelog/109071.yaml                    |   5 +
 .../action/search/SearchRequest.java          |   7 +
 .../search/rescore/RescorerBuilder.java       |  10 +-
 .../integration/LearningToRankRescorerIT.java |  83 +++++++++---
 .../inference/ltr/LearningToRankRescorer.java |  22 ++--
 .../ltr/LearningToRankRescorerBuilder.java    |  46 +++++++
 ...RankRescorerBuilderSerializationTests.java | 120 +++++++++++++++++-
 7 files changed, 253 insertions(+), 40 deletions(-)
 create mode 100644 docs/changelog/109071.yaml

diff --git a/docs/changelog/109071.yaml b/docs/changelog/109071.yaml
new file mode 100644
index 0000000000000..275a5433cc1d8
--- /dev/null
+++ b/docs/changelog/109071.yaml
@@ -0,0 +1,5 @@
+pr: 109071
+summary: Better handling of multiple rescorers clauses with LTR
+area: "Search"
+type: bug
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 6a95eadc92139..4e3fdbc9633b9 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.ShardDocSortField;
 import org.elasticsearch.search.sort.SortBuilder;
@@ -440,6 +441,12 @@ public ActionRequestValidationException validate() {
                     validationException = addValidationError("[rank] requires [explain] is [false]", validationException);
                 }
             }
+            if (source.rescores() != null) {
+                for (@SuppressWarnings("rawtypes")
+                RescorerBuilder rescoreBuilder : source.rescores()) {
+                    validationException = rescoreBuilder.validate(this, validationException);
+                }
+            }
         }
         if (pointInTimeBuilder() != null) {
             if (scroll) {
diff --git a/server/src/main/java/org/elasticsearch/search/rescore/RescorerBuilder.java b/server/src/main/java/org/elasticsearch/search/rescore/RescorerBuilder.java
index 4c42daba22b7a..5cd5a888581c8 100644
--- a/server/src/main/java/org/elasticsearch/search/rescore/RescorerBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/rescore/RescorerBuilder.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.search.rescore;
 
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -118,6 +120,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
+    public ActionRequestValidationException validate(SearchRequest searchRequest, ActionRequestValidationException validationException) {
+        return validationException;
+    }
+
     protected abstract void doXContent(XContentBuilder builder, Params params) throws IOException;
 
     /**
@@ -136,8 +142,8 @@ public final RescoreContext buildContext(SearchExecutionContext context) throws
             assert windowSize != null;
         }
         int finalWindowSize = windowSize == null ? DEFAULT_WINDOW_SIZE : windowSize;
-        RescoreContext rescoreContext = innerBuildContext(finalWindowSize, context);
-        return rescoreContext;
+
+        return innerBuildContext(finalWindowSize, context);
     }
 
     /**
diff --git a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearningToRankRescorerIT.java b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearningToRankRescorerIT.java
index f6aca48a3f493..f17d5bf00297f 100644
--- a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearningToRankRescorerIT.java
+++ b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearningToRankRescorerIT.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.junit.Before;
@@ -17,6 +18,7 @@
 import java.io.IOException;
 import java.util.List;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class LearningToRankRescorerIT extends InferenceTestCase {
@@ -241,33 +243,72 @@ public void testLearningToRankRescoreSmallWindow() throws Exception {
                 "learning_to_rank": { "model_id": "ltr-model" }
               }
             }""");
-        assertThrows(
-            "Rescore window is too small and should be at least the value of from + size but was [2]",
-            ResponseException.class,
-            () -> client().performRequest(request)
+
+        Exception e = assertThrows(ResponseException.class, () -> client().performRequest(request));
+        assertThat(
+            e.getMessage(),
+            containsString("rescorer [window_size] is too small and should be at least the value of [from + size: 4] but was [2]")
         );
     }
 
-    public void testLearningToRankRescorerWithChainedRescorers() throws IOException {
-        Request request = new Request("GET", "store/_search?size=5");
+    public void testLearningToRankRescorerWithFieldCollapsing() throws IOException {
+        Request request = new Request("GET", "store/_search?size=3");
         request.setJsonEntity("""
             {
-               "rescore": [
-                   {
-                     "window_size": 15,
-                     "query": { "rescore_query" : { "script_score": { "query": { "match_all": {} }, "script": { "source": "return 4" } } } }
-                   },
-                   {
-                     "window_size": 25,
-                     "learning_to_rank": { "model_id": "ltr-model" }
-                   },
-                   {
-                     "window_size": 35,
-                     "query": { "rescore_query": { "script_score": { "query": { "match_all": {} }, "script": { "source": "return 20"} } } }
-                   }
-              ]
+             "collapse": {
+                "field": "product"
+              },
+              "rescore": {
+                "window_size": 5,
+                "learning_to_rank": { "model_id": "ltr-model" }
+              }
             }""");
-        assertHitScores(client().performRequest(request), List.of(40.0, 40.0, 37.0, 29.0, 29.0));
+
+        assertHitScores(client().performRequest(request), List.of(20.0, 9.0, 9.0));
+    }
+
+    public void testLearningToRankRescorerWithChainedRescorers() throws IOException {
+
+        String queryTemplate = """
+            {
+              "rescore": [
+                {
+                  "window_size": %d,
+                  "query": { "rescore_query" : { "script_score": { "query": { "match_all": {} }, "script": { "source": "return 4" } } } }
+                },
+                {
+                  "window_size": 5,
+                  "learning_to_rank": { "model_id": "ltr-model" }
+                },
+                {
+                  "window_size": %d,
+                  "query": { "rescore_query": { "script_score": { "query": { "match_all": {} }, "script": { "source": "return 20"} } } }
+                 }
+              ]
+            }""";
+
+        {
+            Request request = new Request("GET", "store/_search?size=4");
+            request.setJsonEntity(Strings.format(queryTemplate, randomIntBetween(2, 10000), randomIntBetween(4, 5)));
+            assertHitScores(client().performRequest(request), List.of(40.0, 40.0, 37.0, 29.0));
+        }
+
+        {
+            int lastRescorerWindowSize = randomIntBetween(6, 10000);
+            Request request = new Request("GET", "store/_search?size=4");
+            request.setJsonEntity(Strings.format(queryTemplate, randomIntBetween(2, 10000), lastRescorerWindowSize));
+
+            Exception e = assertThrows(ResponseException.class, () -> client().performRequest(request));
+            assertThat(
+                e.getMessage(),
+                containsString(
+                    "unable to add a rescorer with [window_size: "
+                        + lastRescorerWindowSize
+                        + "] because a rescorer of type [learning_to_rank]"
+                        + " with a smaller [window_size: 5] has been added before"
+                )
+            );
+        }
     }
 
     private void indexData(String data) throws IOException {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java
index 4e3fa3addaf30..8a310ba2719f2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorer.java
@@ -56,22 +56,16 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r
             throw new IllegalStateException("local model reference is null, missing rewriteAndFetch before rescore phase?");
         }
 
-        if (rescoreContext.getWindowSize() < topDocs.scoreDocs.length) {
-            throw new IllegalArgumentException(
-                "Rescore window is too small and should be at least the value of from + size but was ["
-                    + rescoreContext.getWindowSize()
-                    + "]"
-            );
-        }
-
         LocalModel definition = ltrRescoreContext.regressionModelDefinition;
 
-        // First take top slice of incoming docs, to be rescored:
-        TopDocs topNFirstPass = topN(topDocs, rescoreContext.getWindowSize());
+        // Because scores of the first-pass query and the LTR model are not comparable, there is no way to combine the results.
+        // We will truncate the {@link TopDocs} to the window size so rescoring will be done on the full topDocs.
+        topDocs = topN(topDocs, rescoreContext.getWindowSize());
+
         // Save doc IDs for which rescoring was applied to be used in score explanation
-        Set<Integer> topNDocIDs = Arrays.stream(topNFirstPass.scoreDocs).map(scoreDoc -> scoreDoc.doc).collect(toUnmodifiableSet());
-        rescoreContext.setRescoredDocs(topNDocIDs);
-        ScoreDoc[] hitsToRescore = topNFirstPass.scoreDocs;
+        Set<Integer> topDocIDs = Arrays.stream(topDocs.scoreDocs).map(scoreDoc -> scoreDoc.doc).collect(toUnmodifiableSet());
+        rescoreContext.setRescoredDocs(topDocIDs);
+        ScoreDoc[] hitsToRescore = topDocs.scoreDocs;
         Arrays.sort(hitsToRescore, Comparator.comparingInt(a -> a.doc));
         int hitUpto = 0;
         int readerUpto = -1;
@@ -81,7 +75,7 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r
         LeafReaderContext currentSegment = null;
         boolean changedSegment = true;
         List<FeatureExtractor> featureExtractors = ltrRescoreContext.buildFeatureExtractors(searcher);
-        List<Map<String, Object>> docFeatures = new ArrayList<>(topNDocIDs.size());
+        List<Map<String, Object>> docFeatures = new ArrayList<>(topDocIDs.size());
         int featureSize = featureExtractors.stream().mapToInt(fe -> fe.featureNames().size()).sum();
         while (hitUpto < hitsToRescore.length) {
             final ScoreDoc hit = hitsToRescore[hitUpto];
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java
index 9aa0e75b944fe..f8f9caf365918 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilder.java
@@ -10,6 +10,8 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryRewriteContext;
@@ -26,10 +28,13 @@
 import org.elasticsearch.xpack.ml.inference.loadingservice.LocalModel;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.Function;
 
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
 public class LearningToRankRescorerBuilder extends RescorerBuilder<LearningToRankRescorerBuilder> {
 
     public static final ParseField NAME = new ParseField("learning_to_rank");
@@ -128,6 +133,47 @@ public RescorerBuilder<LearningToRankRescorerBuilder> rewrite(QueryRewriteContex
         return doCoordinatorNodeRewrite(ctx);
     }
 
+    @Override
+    public ActionRequestValidationException validate(SearchRequest searchRequest, ActionRequestValidationException validationException) {
+        validationException = super.validate(searchRequest, validationException);
+
+        int searchRequestPaginationSize = searchRequest.source().from() + searchRequest.source().size();
+
+        if (windowSize() < searchRequestPaginationSize) {
+            return addValidationError(
+                "rescorer [window_size] is too small and should be at least the value of [from + size: "
+                    + searchRequestPaginationSize
+                    + "] but was ["
+                    + windowSize()
+                    + "]",
+                validationException
+            );
+        }
+
+        @SuppressWarnings("rawtypes")
+        List<RescorerBuilder> rescorers = searchRequest.source().rescores();
+        assert rescorers != null && rescorers.contains(this);
+
+        for (int i = rescorers.indexOf(this) + 1; i < rescorers.size(); i++) {
+            RescorerBuilder<?> nextRescorer = rescorers.get(i);
+            int nextRescorerWindowSize = nextRescorer.windowSize() != null ? nextRescorer.windowSize() : DEFAULT_WINDOW_SIZE;
+            if (windowSize() < nextRescorerWindowSize) {
+                return addValidationError(
+                    "unable to add a rescorer with [window_size: "
+                        + nextRescorerWindowSize
+                        + "] because a rescorer of type ["
+                        + getWriteableName()
+                        + "] with a smaller [window_size: "
+                        + windowSize()
+                        + "] has been added before",
+                    validationException
+                );
+            }
+        }
+
+        return validationException;
+    }
+
     /**
      * Here we fetch the stored model inference context, apply the given update, and rewrite.
      *
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilderSerializationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilderSerializationTests.java
index ad57be267295c..2f43e12a2e3c7 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilderSerializationTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilderSerializationTests.java
@@ -8,13 +8,18 @@
 package org.elasticsearch.xpack.ml.inference.ltr;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
@@ -59,6 +64,101 @@ public void testRequiredWindowSize() throws IOException {
         }
     }
 
+    public void testRescoreChainValidation() {
+        {
+            SearchSourceBuilder source = new SearchSourceBuilder().from(10)
+                .size(10)
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 10000)))
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 10000)))
+                .addRescorer(createTestInstance(50))
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 50)))
+                .addRescorer(createTestInstance(50))
+                .addRescorer(createTestInstance(20))
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 20)))
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 20)));
+
+            SearchRequest searchRequest = new SearchRequest().source(source);
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNull(validationErrors);
+        }
+
+        {
+            RescorerBuilder<?> rescorer = createTestInstance(randomIntBetween(2, 19));
+            SearchSourceBuilder source = new SearchSourceBuilder().from(10).size(10).addRescorer(rescorer);
+
+            SearchRequest searchRequest = new SearchRequest().source(source);
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertThat(
+                validationErrors.validationErrors().get(0),
+                equalTo(
+                    "rescorer [window_size] is too small and should be at least the value of [from + size: 20] but was ["
+                        + rescorer.windowSize()
+                        + "]"
+                )
+            );
+        }
+
+        {
+            SearchSourceBuilder source = new SearchSourceBuilder().from(10)
+                .size(10)
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 10000)))
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 10000)))
+                .addRescorer(createTestInstance(50))
+                .addRescorer(createTestInstance(60));
+
+            SearchRequest searchRequest = new SearchRequest().source(source);
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertThat(
+                validationErrors.validationErrors().get(0),
+                equalTo(
+                    "unable to add a rescorer with [window_size: 60] because a rescorer of type [learning_to_rank] "
+                        + "with a smaller [window_size: 50] has been added before"
+                )
+            );
+        }
+
+        {
+            SearchSourceBuilder source = new SearchSourceBuilder().from(10)
+                .size(10)
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 10000)))
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 10000)))
+                .addRescorer(createTestInstance(50))
+                .addRescorer(createQueryRescorerBuilder(60));
+
+            SearchRequest searchRequest = new SearchRequest().source(source);
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertThat(
+                validationErrors.validationErrors().get(0),
+                equalTo(
+                    "unable to add a rescorer with [window_size: 60] because a rescorer of type [learning_to_rank] "
+                        + "with a smaller [window_size: 50] has been added before"
+                )
+            );
+        }
+
+        {
+            SearchSourceBuilder source = new SearchSourceBuilder().size(3)
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 10000)))
+                .addRescorer(createQueryRescorerBuilder(randomIntBetween(2, 10000)))
+                .addRescorer(createTestInstance(5))
+                .addRescorer(createQueryRescorerBuilder(null));
+
+            SearchRequest searchRequest = new SearchRequest().source(source);
+            ActionRequestValidationException validationErrors = searchRequest.validate();
+            assertNotNull(validationErrors);
+            assertThat(
+                validationErrors.validationErrors().get(0),
+                equalTo(
+                    "unable to add a rescorer with [window_size: 10] because a rescorer of type [learning_to_rank] "
+                        + "with a smaller [window_size: 5] has been added before"
+                )
+            );
+        }
+    }
+
     public void testModelIdIsRequired() throws IOException {
         XContentBuilder jsonBuilder = jsonBuilder().startObject();
         if (randomBoolean()) {
@@ -85,8 +185,7 @@ protected Writeable.Reader<LearningToRankRescorerBuilder> instanceReader() {
         return in -> new LearningToRankRescorerBuilder(in, learningToRankService);
     }
 
-    @Override
-    protected LearningToRankRescorerBuilder createTestInstance() {
+    protected LearningToRankRescorerBuilder createTestInstance(int windowSize) {
         LearningToRankRescorerBuilder builder = randomBoolean()
             ? createXContextTestInstance(null)
             : new LearningToRankRescorerBuilder(
@@ -96,11 +195,16 @@ protected LearningToRankRescorerBuilder createTestInstance() {
                 learningToRankService
             );
 
-        builder.windowSize(randomIntBetween(1, 10000));
+        builder.windowSize(windowSize);
 
         return builder;
     }
 
+    @Override
+    protected LearningToRankRescorerBuilder createTestInstance() {
+        return createTestInstance(randomIntBetween(1, 10000));
+    }
+
     @Override
     protected LearningToRankRescorerBuilder createXContextTestInstance(XContentType xContentType) {
         return new LearningToRankRescorerBuilder(randomAlphaOfLength(10), randomBoolean() ? randomParams() : null, learningToRankService)
@@ -187,4 +291,14 @@ protected NamedWriteableRegistry writableRegistry() {
     private static Map<String, Object> randomParams() {
         return randomMap(1, randomIntBetween(1, 10), () -> new Tuple<>(randomIdentifier(), randomIdentifier()));
     }
+
+    private static QueryRescorerBuilder createQueryRescorerBuilder(Integer windowSize) {
+        QueryRescorerBuilder queryRescorer = new QueryRescorerBuilder(mock(QueryBuilder.class));
+
+        if (windowSize != null) {
+            queryRescorer.windowSize(windowSize);
+        }
+
+        return queryRescorer;
+    }
 }

From 7d73389e790983f97f60354365e02b121c0cae51 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Fri, 31 May 2024 21:02:38 +0200
Subject: [PATCH 092/208] Speedup Murmur3HashFunction (#109261)

Allocating a fresh `byte[]` to copy the string to before hashing causes
GBs of allocations in benchmarks like http_logs where many small
documents are indexed append only. We can use a thread-local for the
intermediary byte array and make the char to byte conversion faster via
a `VarHandle`, saving a couple more cycles. This saves those GBs in
allocations and for the http_logs track results in about a 50% speedup
for this method.
---
 .../cluster/routing/Murmur3HashFunction.java  | 19 +++++++++++++++----
 .../elasticsearch/common/util/ByteUtils.java  |  2 ++
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/Murmur3HashFunction.java b/server/src/main/java/org/elasticsearch/cluster/routing/Murmur3HashFunction.java
index 8f0fcb0a1c18c..e6060b8b4b5ba 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/Murmur3HashFunction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/Murmur3HashFunction.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.cluster.routing;
 
 import org.apache.lucene.util.StringHelper;
+import org.elasticsearch.common.util.ByteUtils;
 
 /**
  * Hash function based on the Murmur3 algorithm, which is the default as of Elasticsearch 2.0.
@@ -19,16 +20,26 @@ private Murmur3HashFunction() {
         // no instance
     }
 
+    private static final int MAX_SCRATCH_SIZE = 1024;
+    private static final ThreadLocal<byte[]> scratch = ThreadLocal.withInitial(() -> new byte[MAX_SCRATCH_SIZE]);
+
     public static int hash(String routing) {
-        final byte[] bytesToHash = new byte[routing.length() * 2];
+        assert assertHashWithoutInformationLoss(routing);
+        final int strLen = routing.length();
+        final byte[] bytesToHash = strLen * 2 <= MAX_SCRATCH_SIZE ? scratch.get() : new byte[strLen * 2];
+        for (int i = 0; i < strLen; ++i) {
+            ByteUtils.LITTLE_ENDIAN_CHAR.set(bytesToHash, 2 * i, routing.charAt(i));
+        }
+        return hash(bytesToHash, 0, strLen * 2);
+    }
+
+    private static boolean assertHashWithoutInformationLoss(String routing) {
         for (int i = 0; i < routing.length(); ++i) {
             final char c = routing.charAt(i);
             final byte b1 = (byte) c, b2 = (byte) (c >>> 8);
             assert ((b1 & 0xFF) | ((b2 & 0xFF) << 8)) == c; // no information loss
-            bytesToHash[i * 2] = b1;
-            bytesToHash[i * 2 + 1] = b2;
         }
-        return hash(bytesToHash, 0, bytesToHash.length);
+        return true;
     }
 
     public static int hash(byte[] bytes, int offset, int length) {
diff --git a/server/src/main/java/org/elasticsearch/common/util/ByteUtils.java b/server/src/main/java/org/elasticsearch/common/util/ByteUtils.java
index 06be38ce34e5d..a4d1b4e5de709 100644
--- a/server/src/main/java/org/elasticsearch/common/util/ByteUtils.java
+++ b/server/src/main/java/org/elasticsearch/common/util/ByteUtils.java
@@ -17,6 +17,8 @@
 public enum ByteUtils {
     ;
 
+    public static final VarHandle LITTLE_ENDIAN_CHAR = MethodHandles.byteArrayViewVarHandle(char[].class, ByteOrder.LITTLE_ENDIAN);
+
     public static final VarHandle LITTLE_ENDIAN_INT = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN);
 
     public static final VarHandle LITTLE_ENDIAN_LONG = MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.LITTLE_ENDIAN);

From af4bacdd69dbc9ed02916ca2c843548b7469983c Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Fri, 31 May 2024 16:52:40 -0400
Subject: [PATCH 093/208] Mute
 org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
 org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT #109266

---
 muted-tests.yml | 109 +++++++++++++++++++++++++-----------------------
 1 file changed, 56 insertions(+), 53 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index c7a18b527e8ce..a3becc570d98c 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -1,57 +1,60 @@
 tests:
-  - class: "org.elasticsearch.xpack.transform.transforms.scheduling.MonotonicClockTests"
-    issue: "https://github.com/elastic/elasticsearch/issues/108529"
-  - class: "org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterTests"
-    issue: "https://github.com/elastic/elasticsearch/issues/108649"
-    method: "testManyRandomDocs"
-  - class: "org.elasticsearch.cluster.coordination.CoordinatorVotingConfigurationTests"
-    issue: "https://github.com/elastic/elasticsearch/issues/108729"
-    method: "testClusterUUIDLogging"
-  - class: "org.elasticsearch.xpack.core.ssl.SSLConfigurationReloaderTests"
-    issue: "https://github.com/elastic/elasticsearch/issues/108774"
-    method: "testReloadingKeyStore"
-  - class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
-    issue: "https://github.com/elastic/elasticsearch/issues/108808"
-    method: "test {k8s-metrics.MetricsWithAggs}"
-  - class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
-    issue: "https://github.com/elastic/elasticsearch/issues/108809"
-    method: "test {k8s-metrics.MetricsWithoutAggs}"
-  - class: "org.elasticsearch.xpack.textstructure.structurefinder.TimestampFormatFinderTests"
-    issue: "https://github.com/elastic/elasticsearch/issues/108855"
-    method: "testGuessIsDayFirstFromLocale"
-  - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
-    issue: "https://github.com/elastic/elasticsearch/issues/108857"
-    method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
-  - class: "org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT"
-    issue: "https://github.com/elastic/elasticsearch/issues/108950"
-    method: "test {p0=health/10_basic/cluster health basic test}"
-  - class: "org.elasticsearch.upgrades.SearchStatesIT"
-    issue: "https://github.com/elastic/elasticsearch/issues/108991"
-    method: "testCanMatch"
-  - class: "org.elasticsearch.upgrades.AggregationsIT"
-    issue: "https://github.com/elastic/elasticsearch/issues/108992"
-    method: "testHistogram"
-  - class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT"
-    issue: "https://github.com/elastic/elasticsearch/issues/108993"
-    method: "testTrainedModelInference"
-  - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
-    issue: "https://github.com/elastic/elasticsearch/issues/109188"
-    method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
-  - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
-    issue: "https://github.com/elastic/elasticsearch/issues/109189"
-    method: "test {p0=esql/70_locale/Date format with Italian locale}"
-  - class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
-    issue: "https://github.com/elastic/elasticsearch/issues/109200"
-    method: "test {p0=esql/70_locale/Date format with Italian locale}"
-  - class: org.elasticsearch.blobcache.common.SparseFileTrackerTests
-    method: testCallsListenerWhenRangeIsAvailable
-    issue: https://github.com/elastic/elasticsearch/issues/109259
-  - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
-    method: test {yaml=reference/esql/esql-async-query-api/line_17}
-    issue: https://github.com/elastic/elasticsearch/issues/109260
-  - class: org.elasticsearch.xpack.ccr.LocalIndexFollowingIT
-    method: testRemoveRemoteConnection
-    issue: https://github.com/elastic/elasticsearch/issues/109163
+- class: "org.elasticsearch.xpack.transform.transforms.scheduling.MonotonicClockTests"
+  issue: "https://github.com/elastic/elasticsearch/issues/108529"
+- class: "org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterTests"
+  issue: "https://github.com/elastic/elasticsearch/issues/108649"
+  method: "testManyRandomDocs"
+- class: "org.elasticsearch.cluster.coordination.CoordinatorVotingConfigurationTests"
+  issue: "https://github.com/elastic/elasticsearch/issues/108729"
+  method: "testClusterUUIDLogging"
+- class: "org.elasticsearch.xpack.core.ssl.SSLConfigurationReloaderTests"
+  issue: "https://github.com/elastic/elasticsearch/issues/108774"
+  method: "testReloadingKeyStore"
+- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/108808"
+  method: "test {k8s-metrics.MetricsWithAggs}"
+- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/108809"
+  method: "test {k8s-metrics.MetricsWithoutAggs}"
+- class: "org.elasticsearch.xpack.textstructure.structurefinder.TimestampFormatFinderTests"
+  issue: "https://github.com/elastic/elasticsearch/issues/108855"
+  method: "testGuessIsDayFirstFromLocale"
+- class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/108857"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+- class: "org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/108950"
+  method: "test {p0=health/10_basic/cluster health basic test}"
+- class: "org.elasticsearch.upgrades.SearchStatesIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/108991"
+  method: "testCanMatch"
+- class: "org.elasticsearch.upgrades.AggregationsIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/108992"
+  method: "testHistogram"
+- class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/108993"
+  method: "testTrainedModelInference"
+- class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109188"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+- class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109189"
+  method: "test {p0=esql/70_locale/Date format with Italian locale}"
+- class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109200"
+  method: "test {p0=esql/70_locale/Date format with Italian locale}"
+- class: org.elasticsearch.blobcache.common.SparseFileTrackerTests
+  method: testCallsListenerWhenRangeIsAvailable
+  issue: https://github.com/elastic/elasticsearch/issues/109259
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/esql/esql-async-query-api/line_17}
+  issue: https://github.com/elastic/elasticsearch/issues/109260
+- class: org.elasticsearch.xpack.ccr.LocalIndexFollowingIT
+  method: testRemoveRemoteConnection
+  issue: https://github.com/elastic/elasticsearch/issues/109163
+- class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
+  method: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
+  issue: https://github.com/elastic/elasticsearch/issues/109266
 
 # Examples:
 #

From 18219ca600a46973a88fffa10f90e0ecebadd4b4 Mon Sep 17 00:00:00 2001
From: Mikhail Berezovskiy <mikhail@elastic.co>
Date: Fri, 31 May 2024 14:50:16 -0700
Subject: [PATCH 094/208] Verify repository before cluster update (#108531)

* add verification before cluster update
* dont verify repo in testRequestStats
---
 .../s3/S3BlobStoreRepositoryTests.java        |  4 +-
 .../snapshots/RepositoriesIT.java             |  6 +-
 .../SharedClusterSnapshotRestoreIT.java       |  1 +
 .../repositories/RepositoriesService.java     | 28 ++++++-
 .../RepositoriesServiceTests.java             | 77 +++++++++++++++----
 ...ESMockAPIBasedRepositoryIntegTestCase.java |  4 +-
 .../org/elasticsearch/test/ESTestCase.java    |  9 +++
 7 files changed, 104 insertions(+), 25 deletions(-)

diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
index 030f791feee16..88f0e01db3e6a 100644
--- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -184,7 +184,7 @@ public void testRequestStats() throws Exception {
     }
 
     public void testAbortRequestStats() throws Exception {
-        final String repository = createRepository(randomRepositoryName());
+        final String repository = createRepository(randomRepositoryName(), false);
 
         final String index = "index-no-merges";
         createIndex(index, 1, 0);
@@ -228,7 +228,7 @@ public void testAbortRequestStats() throws Exception {
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101608")
     public void testMetrics() throws Exception {
         // Create the repository and perform some activities
-        final String repository = createRepository(randomRepositoryName());
+        final String repository = createRepository(randomRepositoryName(), false);
         final String index = "index-no-merges";
         createIndex(index, 1, 0);
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
index 313c395b6bc24..80ded243d3fb2 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
@@ -302,7 +302,11 @@ public void testRepositoryConflict() throws Exception {
         logger.info("--> try updating the repository, should fail because the deletion of the snapshot is in progress");
         RepositoryConflictException e2 = expectThrows(
             RepositoryConflictException.class,
-            clusterAdmin().preparePutRepository(repo).setType("mock").setSettings(Settings.builder().put("location", randomRepoPath()))
+            clusterAdmin().preparePutRepository(repo)
+                // if "true" will deadlock on snapshot thread pool, we are running with single thread which is busy at the moment
+                .setVerify(false)
+                .setType("mock")
+                .setSettings(Settings.builder().put("location", randomRepoPath()))
         );
         assertThat(e2.status(), equalTo(RestStatus.CONFLICT));
         assertThat(e2.getMessage(), containsString("trying to modify or unregister repository that is currently used"));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 2b7c91213b9b7..005604b92a723 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -959,6 +959,7 @@ public void testDeleteRepositoryWhileSnapshotting() throws Exception {
             client.admin()
                 .cluster()
                 .preparePutRepository("test-repo")
+                .setVerify(false)
                 .setType("fs")
                 .setSettings(Settings.builder().put("location", repositoryLocation.resolve("test")))
                 .get();
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index e0b2aa61d9c1c..4884299bd1b4e 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -161,10 +161,7 @@ record RegisterRepositoryTaskResult(AcknowledgedResponse ackResponse, boolean ch
         SubscribableListener
 
             // Trying to create the new repository on master to make sure it works
-            .<Void>newForked(validationStep -> ActionListener.completeWith(validationStep, () -> {
-                validateRepositoryCanBeCreated(request);
-                return null;
-            }))
+            .<Void>newForked(validationStep -> validatePutRepositoryRequest(request, validationStep))
 
             // When publication has completed (and all acks received or timed out) then verify the repository.
             // (if acks timed out then acknowledgementStep completes before the master processes this cluster state, hence why we have
@@ -353,6 +350,29 @@ public void validateRepositoryCanBeCreated(final PutRepositoryRequest request) {
         closeRepository(createRepository(newRepositoryMetadata));
     }
 
+    private void validatePutRepositoryRequest(final PutRepositoryRequest request, ActionListener<Void> resultListener) {
+        final RepositoryMetadata newRepositoryMetadata = new RepositoryMetadata(request.name(), request.type(), request.settings());
+        try {
+            final var repository = createRepository(newRepositoryMetadata);
+            if (request.verify()) {
+                // verify repository on local node only, different from verifyRepository method that runs on other cluster nodes
+                threadPool.executor(ThreadPool.Names.SNAPSHOT)
+                    .execute(ActionRunnable.run(ActionListener.runBefore(resultListener, () -> closeRepository(repository)), () -> {
+                        final var token = repository.startVerification();
+                        if (token != null) {
+                            repository.verify(token, clusterService.localNode());
+                            repository.endVerification(token);
+                        }
+                    }));
+            } else {
+                closeRepository(repository);
+                resultListener.onResponse(null);
+            }
+        } catch (Exception e) {
+            resultListener.onFailure(e);
+        }
+    }
+
     private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) {
         submitUnbatchedTask(clusterService, source, task);
     }
diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
index bfbd92bd5df22..e156fae19f2ef 100644
--- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
@@ -48,8 +48,6 @@
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -65,28 +63,16 @@
 
 public class RepositoriesServiceTests extends ESTestCase {
 
-    private static ThreadPool threadPool;
-
     private ClusterService clusterService;
     private RepositoriesService repositoriesService;
-
-    @BeforeClass
-    public static void createThreadPool() {
-        threadPool = new TestThreadPool(RepositoriesService.class.getName());
-    }
-
-    @AfterClass
-    public static void terminateThreadPool() {
-        if (threadPool != null) {
-            threadPool.shutdownNow();
-            threadPool = null;
-        }
-    }
+    private ThreadPool threadPool;
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
 
+        threadPool = new TestThreadPool(RepositoriesService.class.getName());
+
         final TransportService transportService = new TransportService(
             Settings.EMPTY,
             mock(Transport.class),
@@ -113,6 +99,8 @@ public void setUp() throws Exception {
             TestRepository::new,
             UnstableRepository.TYPE,
             UnstableRepository::new,
+            VerificationFailRepository.TYPE,
+            VerificationFailRepository::new,
             MeteredRepositoryTypeA.TYPE,
             metadata -> new MeteredRepositoryTypeA(metadata, clusterService),
             MeteredRepositoryTypeB.TYPE,
@@ -135,6 +123,10 @@ public void setUp() throws Exception {
     @Override
     public void tearDown() throws Exception {
         super.tearDown();
+        if (threadPool != null) {
+            threadPool.shutdownNow();
+            threadPool = null;
+        }
         clusterService.stop();
         repositoriesService.stop();
     }
@@ -181,6 +173,44 @@ public void testRegisterRejectsInvalidRepositoryNames() {
         }
     }
 
+    public void testPutRepositoryVerificationFails() {
+        var repoName = randomAlphaOfLengthBetween(10, 25);
+        var request = new PutRepositoryRequest().name(repoName).type(VerificationFailRepository.TYPE).verify(true);
+        var resultListener = new SubscribableListener<AcknowledgedResponse>();
+        repositoriesService.registerRepository(request, resultListener);
+        var failure = safeAwaitFailure(resultListener);
+        assertThat(failure, isA(RepositoryVerificationException.class));
+        // also make sure that cluster state does not include failed repo
+        assertThrows(RepositoryMissingException.class, () -> { repositoriesService.repository(repoName); });
+    }
+
+    public void testPutRepositoryVerificationFailsOnExisting() {
+        var repoName = randomAlphaOfLengthBetween(10, 25);
+        var request = new PutRepositoryRequest().name(repoName).type(TestRepository.TYPE).verify(true);
+        var resultListener = new SubscribableListener<AcknowledgedResponse>();
+        repositoriesService.registerRepository(request, resultListener);
+        var ackResponse = safeAwait(resultListener);
+        assertTrue(ackResponse.isAcknowledged());
+
+        // try to update existing repository with faulty repo and make sure it is not applied
+        request = new PutRepositoryRequest().name(repoName).type(VerificationFailRepository.TYPE).verify(true);
+        resultListener = new SubscribableListener<>();
+        repositoriesService.registerRepository(request, resultListener);
+        var failure = safeAwaitFailure(resultListener);
+        assertThat(failure, isA(RepositoryVerificationException.class));
+        var repository = repositoriesService.repository(repoName);
+        assertEquals(repository.getMetadata().type(), TestRepository.TYPE);
+    }
+
+    public void testPutRepositorySkipVerification() {
+        var repoName = randomAlphaOfLengthBetween(10, 25);
+        var request = new PutRepositoryRequest().name(repoName).type(VerificationFailRepository.TYPE).verify(false);
+        var resultListener = new SubscribableListener<AcknowledgedResponse>();
+        repositoriesService.registerRepository(request, resultListener);
+        var ackResponse = safeAwait(resultListener);
+        assertTrue(ackResponse.isAcknowledged());
+    }
+
     public void testRepositoriesStatsCanHaveTheSameNameAndDifferentTypeOverTime() {
         String repoName = "name";
         expectThrows(RepositoryMissingException.class, () -> repositoriesService.repository(repoName));
@@ -502,6 +532,19 @@ private UnstableRepository(RepositoryMetadata metadata) {
         }
     }
 
+    private static class VerificationFailRepository extends TestRepository {
+        public static final String TYPE = "verify-fail";
+
+        private VerificationFailRepository(RepositoryMetadata metadata) {
+            super(metadata);
+        }
+
+        @Override
+        public String startVerification() {
+            throw new RepositoryVerificationException(TYPE, "failed to validate repository");
+        }
+    }
+
     private static class MeteredRepositoryTypeA extends MeteredBlobStoreRepository {
         private static final String TYPE = "type-a";
         private static final RepositoryStats STATS = new RepositoryStats(Map.of("GET", 10L));
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java
index b6415eea7db2c..22bbfad3cfb70 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java
@@ -181,7 +181,9 @@ public final void testSnapshotWithLargeSegmentFiles() throws Exception {
     }
 
     public void testRequestStats() throws Exception {
-        final String repository = createRepository(randomRepositoryName());
+        // need to use verify=false, because the verification process on master makes extra calls on placeholder repo
+        // hence impacting http metrics and failing test
+        final String repository = createRepository(randomRepositoryName(), false);
         final String index = "index-no-merges";
         createIndex(index, 1, 0);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 02be27b92a1eb..42cc719a904cd 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.RequestBuilder;
+import org.elasticsearch.action.support.ActionTestUtils;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.bootstrap.BootstrapForTesting;
@@ -2187,6 +2188,14 @@ public static <T> T safeGet(Future<T> future) {
         }
     }
 
+    public static Exception safeAwaitFailure(SubscribableListener<?> listener) {
+        return safeAwait(
+            SubscribableListener.newForked(
+                exceptionListener -> listener.addListener(ActionTestUtils.assertNoSuccessListener(exceptionListener::onResponse))
+            )
+        );
+    }
+
     public static void safeSleep(TimeValue timeValue) {
         safeSleep(timeValue.millis());
     }

From a5092e2a9b67928ab6fc1a601b2c6c6d5318ea26 Mon Sep 17 00:00:00 2001
From: Henning Andersen <33268011+henningandersen@users.noreply.github.com>
Date: Sat, 1 Jun 2024 07:56:39 +0200
Subject: [PATCH 095/208] Use proper executor for failing requests when
 connection closes (#109236)

Now use the executor declared by the handler rather than generic, since using
generic is trappy wrt. deadlocks.

Closes #109225
---
 docs/changelog/109236.yaml                    |  6 ++
 .../transport/TransportServiceIT.java         | 26 ++++++
 .../common/settings/ClusterSettings.java      |  4 +-
 .../ForkingResponseHandlerRunnable.java       | 12 ++-
 .../transport/TransportService.java           | 68 +++++++++------
 .../TransportServiceLifecycleTests.java       | 83 ++++++++++++++++++-
 6 files changed, 169 insertions(+), 30 deletions(-)
 create mode 100644 docs/changelog/109236.yaml
 create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/transport/TransportServiceIT.java

diff --git a/docs/changelog/109236.yaml b/docs/changelog/109236.yaml
new file mode 100644
index 0000000000000..e2eb917ea0343
--- /dev/null
+++ b/docs/changelog/109236.yaml
@@ -0,0 +1,6 @@
+pr: 109236
+summary: Use proper executor for failing requests when connection closes
+area: Network
+type: bug
+issues:
+ - 109225
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/transport/TransportServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/transport/TransportServiceIT.java
new file mode 100644
index 0000000000000..e6fc8e90f6676
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/transport/TransportServiceIT.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.transport;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+public class TransportServiceIT extends ESSingleNodeTestCase {
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder()
+            .put(super.nodeSettings())
+            .put(TransportService.ENABLE_STACK_OVERFLOW_AVOIDANCE.getKey(), randomBoolean())
+            .build();
+    }
+
+    public void testNodeStartsWithSetting() {
+        // just check that the node starts with the setting set, i.e., that the setting is registered.
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 3493206e00bf6..2ea0cc60b5afd 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -124,6 +124,7 @@
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.RemoteConnectionStrategy;
 import org.elasticsearch.transport.SniffConnectionStrategy;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
@@ -590,6 +591,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         IngestSettings.GROK_WATCHDOG_MAX_EXECUTION_TIME,
         TDigestExecutionHint.SETTING,
         MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT_SETTING,
-        MergePolicyConfig.DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT_SETTING
+        MergePolicyConfig.DEFAULT_MAX_TIME_BASED_MERGED_SEGMENT_SETTING,
+        TransportService.ENABLE_STACK_OVERFLOW_AVOIDANCE
     ).filter(Objects::nonNull).collect(Collectors.toSet());
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/ForkingResponseHandlerRunnable.java b/server/src/main/java/org/elasticsearch/transport/ForkingResponseHandlerRunnable.java
index 121dc433862f0..36fdb4355dceb 100644
--- a/server/src/main/java/org/elasticsearch/transport/ForkingResponseHandlerRunnable.java
+++ b/server/src/main/java/org/elasticsearch/transport/ForkingResponseHandlerRunnable.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
 
+import java.util.concurrent.Executor;
+
 import static org.elasticsearch.core.Strings.format;
 
 /**
@@ -30,7 +32,15 @@ abstract class ForkingResponseHandlerRunnable extends AbstractRunnable {
     private final TransportException transportException;
 
     ForkingResponseHandlerRunnable(TransportResponseHandler<?> handler, @Nullable TransportException transportException) {
-        assert handler.executor() != EsExecutors.DIRECT_EXECUTOR_SERVICE : "forking handler required, but got " + handler;
+        this(handler, transportException, handler.executor());
+    }
+
+    ForkingResponseHandlerRunnable(
+        TransportResponseHandler<?> handler,
+        @Nullable TransportException transportException,
+        Executor executorUsed
+    ) {
+        assert executorUsed != EsExecutors.DIRECT_EXECUTOR_SERVICE : "forking handler required, but got " + handler;
         this.handler = handler;
         this.transportException = transportException;
     }
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java
index 2ade579f216e4..c3d53855a9c75 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -90,6 +91,17 @@ public class TransportService extends AbstractLifecycleComponent
     public static final String DIRECT_RESPONSE_PROFILE = ".direct";
     public static final String HANDSHAKE_ACTION_NAME = "internal:transport/handshake";
 
+    /**
+     * Undocumented on purpose, may be removed at any time. Only use this if instructed to do so, can have other unintended consequences
+     * including deadlocks.
+     */
+    public static final Setting<Boolean> ENABLE_STACK_OVERFLOW_AVOIDANCE = Setting.boolSetting(
+        "transport.enable_stack_protection",
+        false,
+        Setting.Property.NodeScope,
+        Setting.Property.Deprecated
+    );
+
     private final AtomicBoolean handleIncomingRequests = new AtomicBoolean();
     private final DelegatingTransportMessageListener messageListener = new DelegatingTransportMessageListener();
     protected final Transport transport;
@@ -105,6 +117,8 @@ public class TransportService extends AbstractLifecycleComponent
 
     private final PendingDirectHandlers pendingDirectHandlers = new PendingDirectHandlers();
 
+    private final boolean enableStackOverflowAvoidance;
+
     // An LRU (don't really care about concurrency here) that holds the latest timed out requests so if they
     // do show up, we can print more descriptive information about them
     final Map<Long, TimeoutInfoHolder> timeoutInfoHandlers = Collections.synchronizedMap(new LinkedHashMap<>(100, .75F, true) {
@@ -281,6 +295,7 @@ public TransportService(
         this.interceptor = transportInterceptor;
         this.asyncSender = interceptor.interceptSender(this::sendRequestInternal);
         this.remoteClusterClient = DiscoveryNode.isRemoteClusterClient(settings);
+        this.enableStackOverflowAvoidance = ENABLE_STACK_OVERFLOW_AVOIDANCE.get(settings);
         remoteClusterService = new RemoteClusterService(settings, this);
         responseHandlers = transport.getResponseHandlers();
         if (clusterSettings != null) {
@@ -1342,36 +1357,35 @@ public void onConnectionClosed(Transport.Connection connection) {
             return;
         }
 
-        // Callback that an exception happened, but on a different thread since we don't want handlers to worry about stack overflows.
-        final var executor = threadPool.generic();
-        assert executor.isShutdown() == false : "connections should all be closed before threadpool shuts down";
-        executor.execute(new AbstractRunnable() {
-            @Override
-            public void doRun() {
-                for (Transport.ResponseContext<?> holderToNotify : pruned) {
-                    if (tracerLog.isTraceEnabled() && shouldTraceAction(holderToNotify.action())) {
-                        tracerLog.trace(
-                            "[{}][{}] pruning request because connection to node [{}] closed",
-                            holderToNotify.requestId(),
-                            holderToNotify.action(),
-                            connection.getNode()
-                        );
-                    }
-                    holderToNotify.handler().handleException(new NodeDisconnectedException(connection.getNode(), holderToNotify.action()));
-                }
+        for (Transport.ResponseContext<?> holderToNotify : pruned) {
+            if (tracerLog.isTraceEnabled() && shouldTraceAction(holderToNotify.action())) {
+                tracerLog.trace(
+                    "[{}][{}] pruning request because connection to node [{}] closed",
+                    holderToNotify.requestId(),
+                    holderToNotify.action(),
+                    connection.getNode()
+                );
             }
-
-            @Override
-            public void onFailure(Exception e) {
-                assert false : e;
-                logger.warn(() -> "failed to notify response handler on connection close [" + connection + "]", e);
+            NodeDisconnectedException exception = new NodeDisconnectedException(connection.getNode(), holderToNotify.action());
+
+            TransportResponseHandler<?> handler = holderToNotify.handler();
+            // we used to fork to a different thread always to avoid stack overflows, but we avoid doing that now, expecting handlers
+            // to handle that themselves instead.
+            var executor = handler.executor();
+            if (executor == EsExecutors.DIRECT_EXECUTOR_SERVICE && enableStackOverflowAvoidance) {
+                executor = threadPool.generic();
             }
-
-            @Override
-            public String toString() {
-                return "onConnectionClosed(" + connection.getNode() + ")";
+            if (executor == EsExecutors.DIRECT_EXECUTOR_SERVICE) {
+                handler.handleException(exception);
+            } else {
+                executor.execute(new ForkingResponseHandlerRunnable(handler, exception, executor) {
+                    @Override
+                    protected void doRun() {
+                        handler.handleException(exception);
+                    }
+                });
             }
-        });
+        }
     }
 
     final class TimeoutHandler implements Runnable {
diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java
index 062cc71c9172d..7b3de5f142597 100644
--- a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java
@@ -10,7 +10,9 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.support.ActionTestUtils;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
@@ -45,6 +47,8 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.startsWith;
 
 public class TransportServiceLifecycleTests extends ESTestCase {
@@ -217,6 +221,60 @@ public void testInternalSendExceptionCompletesHandlerOnCallingThreadIfTransportS
         assertThat(getSendRequestException(future, NodeClosedException.class).getMessage(), startsWith("node closed"));
     }
 
+    public void testOnConnectionClosedUsesHandlerExecutor() {
+        String executorName = randomFrom(TestNode.EXECUTOR_NAMES);
+        String expectedExecutor = (executorName.equals(Executors.DIRECT) ? null : executorName);
+        boolean withSetting = randomBoolean();
+        Settings settings = withSetting
+            ? Settings.builder().put(TransportService.ENABLE_STACK_OVERFLOW_AVOIDANCE.getKey(), false).build()
+            : Settings.EMPTY;
+        onConnectionClosedUsesHandlerExecutor(settings, executorName, expectedExecutor);
+        if (withSetting) {
+            assertWarnings(
+                "[transport.enable_stack_protection] setting was deprecated in Elasticsearch and will be removed in a future release."
+            );
+        }
+    }
+
+    public void testOnConnectionCloseStackOverflowAvoidance() {
+        onConnectionClosedUsesHandlerExecutor(
+            Settings.builder().put(TransportService.ENABLE_STACK_OVERFLOW_AVOIDANCE.getKey(), true).build(),
+            Executors.DIRECT,
+            ThreadPool.Names.GENERIC
+        );
+        assertWarnings(
+            "[transport.enable_stack_protection] setting was deprecated in Elasticsearch and will be removed in a future release."
+        );
+    }
+
+    private void onConnectionClosedUsesHandlerExecutor(Settings settings, String executorName, String expectedExecutor) {
+        try (var nodeA = new TestNode("node-A", settings)) {
+            final var testThread = Thread.currentThread();
+            final var future = new PlainActionFuture<Exception>();
+            Executor executor = nodeA.getExecutor(executorName);
+            Transport.Connection connection = nodeA.getDevNullConnection();
+            nodeA.transportService.sendRequest(
+                connection,
+                TestNode.randomActionName(random()),
+                new TransportRequest.Empty(),
+                TransportRequestOptions.EMPTY,
+                new ActionListenerResponseHandler<>(
+                    ActionListener.assertOnce(ActionTestUtils.assertNoSuccessListener(future::onResponse).delegateResponse((l, e) -> {
+                        assertThat(EsExecutors.executorName(Thread.currentThread()), equalTo(expectedExecutor));
+                        if (expectedExecutor == null) {
+                            assertSame(testThread, Thread.currentThread());
+                        }
+                        l.onFailure(e);
+                    })),
+                    unusedReader(),
+                    executor
+                )
+            );
+            nodeA.transportService.onConnectionClosed(connection);
+            assertThat(safeGet(future), instanceOf(NodeDisconnectedException.class));
+        }
+    }
+
     private static <T> Writeable.Reader<T> unusedReader() {
         return in -> fail(null, "should not be used");
     }
@@ -251,6 +309,10 @@ private static class TestNode implements Releasable {
         final TransportService transportService;
 
         TestNode(String nodeName) {
+            this(nodeName, Settings.EMPTY);
+        }
+
+        TestNode(String nodeName, Settings settings) {
             threadPool = new TestThreadPool(
                 nodeName,
                 new ScalingExecutorBuilder(Executors.SCALING_DROP_ON_SHUTDOWN, 3, 3, TimeValue.timeValueSeconds(60), false),
@@ -281,7 +343,7 @@ public ExecutorService executor(String name) {
             };
             final var tcpTransport = MockTransportService.newMockTransport(Settings.EMPTY, TransportVersion.current(), threadPool);
             transportService = new TransportService(
-                Settings.EMPTY,
+                settings,
                 tcpTransport,
                 threadPool,
                 TransportService.NOOP_TRANSPORT_INTERCEPTOR,
@@ -351,6 +413,25 @@ public TransportVersion getTransportVersion() {
                 }
             };
         }
+
+        Transport.Connection getDevNullConnection() {
+            return new CloseableConnection() {
+                @Override
+                public DiscoveryNode getNode() {
+                    return transportService.getLocalNode();
+                }
+
+                @Override
+                public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) {
+                    // going nowhere
+                }
+
+                @Override
+                public TransportVersion getTransportVersion() {
+                    return TransportVersion.current();
+                }
+            };
+        }
     }
 
 }

From 55101cf55e945f29a1853fee93acb22b12b876a1 Mon Sep 17 00:00:00 2001
From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com>
Date: Sun, 2 Jun 2024 12:57:16 +0100
Subject: [PATCH 096/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109273

---
 muted-tests.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index a3becc570d98c..9d6b663bc60b0 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -21,7 +21,8 @@ tests:
   method: "testGuessIsDayFirstFromLocale"
 - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108857"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
+    \ dependent mappings / dates}"
 - class: "org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108950"
   method: "test {p0=health/10_basic/cluster health basic test}"
@@ -36,7 +37,8 @@ tests:
   method: "testTrainedModelInference"
 - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109188"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
+    \ dependent mappings / dates}"
 - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109189"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
@@ -55,6 +57,9 @@ tests:
 - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
   method: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/109266
+- class: "org.elasticsearch.xpack.sql.qa.single_node.JdbcDocCsvSpecIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109273"
+  method: "test {docs.testFilterToday}"
 
 # Examples:
 #

From 89113e9b3c2fc73b5d94213285197ecc39ae94b5 Mon Sep 17 00:00:00 2001
From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com>
Date: Mon, 3 Jun 2024 08:30:36 +0100
Subject: [PATCH 097/208] Fix sql JdbcDocCsvSpecIT test {docs.testFilterToday}
 failing (#109274)

This commit fixes an SQL test that hardcodes expected results based on today's date. The docs.testFilterToday test needs to be updated since June 2 2024, because Anneke's hire-date of 1989-06-02 (35 years ago as of time of writing) affects the expected output.
---
 muted-tests.yml                                          | 9 ++-------
 .../sql/qa/server/src/main/resources/docs/docs.csv-spec  | 2 +-
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 9d6b663bc60b0..a3becc570d98c 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -21,8 +21,7 @@ tests:
   method: "testGuessIsDayFirstFromLocale"
 - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108857"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
-    \ dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
 - class: "org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108950"
   method: "test {p0=health/10_basic/cluster health basic test}"
@@ -37,8 +36,7 @@ tests:
   method: "testTrainedModelInference"
 - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109188"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
-    \ dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
 - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109189"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
@@ -57,9 +55,6 @@ tests:
 - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
   method: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/109266
-- class: "org.elasticsearch.xpack.sql.qa.single_node.JdbcDocCsvSpecIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109273"
-  method: "test {docs.testFilterToday}"
 
 # Examples:
 #
diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec
index 8e3853cf187db..60e81be43cc96 100644
--- a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec
+++ b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec
@@ -3351,9 +3351,9 @@ SELECT first_name FROM emp WHERE hire_date > TODAY() - INTERVAL 35 YEARS ORDER B
 ------------
 Alejandro
 Amabile
-Anneke
 Anoosh
 Basil
+Bojan
 // end::filterToday
 ;
 

From 7ef52ec3a34190a1dd38e40b9e71a541105c42fb Mon Sep 17 00:00:00 2001
From: Yang Wang <yang.wang@elastic.co>
Date: Mon, 3 Jun 2024 20:08:48 +1000
Subject: [PATCH 098/208] [Test] Unmute testCallsListenerWhenRangeIsAvailable
 (#109277)

The failure is fixed by #109239
Relates: #109259
---
 muted-tests.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index a3becc570d98c..de8481fcbb02b 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -43,9 +43,6 @@ tests:
 - class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109200"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
-- class: org.elasticsearch.blobcache.common.SparseFileTrackerTests
-  method: testCallsListenerWhenRangeIsAvailable
-  issue: https://github.com/elastic/elasticsearch/issues/109259
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/esql/esql-async-query-api/line_17}
   issue: https://github.com/elastic/elasticsearch/issues/109260

From 66a1df3f277b6ee9dce8d2b98890f69f03e335c8 Mon Sep 17 00:00:00 2001
From: Yang Wang <yang.wang@elastic.co>
Date: Mon, 3 Jun 2024 20:50:23 +1000
Subject: [PATCH 099/208] [Test] Fix FrozenIndexInputTests testRandomReads
 (#109280)

Since #106794, FrozenIndexInput#clone can return a ByteArrayIndexInput
if the content is fully covered by its internal buffer. This PR ensures
the assertion handle this case.

Resolves: #109171
---
 .../store/input/FrozenIndexInputTests.java           | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java
index ca4ad51ecb819..81e9c06a149b9 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.blobcache.BlobCacheMetrics;
 import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.blobcache.shared.SharedBytes;
+import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Tuple;
@@ -44,6 +45,7 @@
 import static org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService.resolveSnapshotCache;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.isA;
 import static org.hamcrest.Matchers.not;
 
 public class FrozenIndexInputTests extends AbstractSearchableSnapshotsTestCase {
@@ -137,9 +139,13 @@ public void testRandomReads() throws IOException {
 
             // validate clone copies cache file object
             indexInput.seek(randomLongBetween(0, fileData.length - 1));
-            FrozenIndexInput clone = (FrozenIndexInput) indexInput.clone();
-            assertThat(clone.cacheFile(), not(equalTo(((FrozenIndexInput) indexInput).cacheFile())));
-            assertThat(clone.getFilePointer(), equalTo(indexInput.getFilePointer()));
+            final IndexInput indexInputClone = indexInput.clone();
+            if (indexInputClone instanceof FrozenIndexInput clone) {
+                assertThat(clone.cacheFile(), not(equalTo(((FrozenIndexInput) indexInput).cacheFile())));
+                assertThat(clone.getFilePointer(), equalTo(indexInput.getFilePointer()));
+            } else {
+                assertThat(indexInputClone, isA(ByteArrayIndexInput.class));
+            }
 
             indexInput.close();
         }

From 6704a788d1cbabbf94c728e20e3245bf74062df5 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 3 Jun 2024 13:52:25 +0200
Subject: [PATCH 100/208] Simplify call to Lucene's IndexWrite in
 InternalEngine (#109270)

No need to special case the single doc case here, Lucene will simply
re-wrap the single document in a list and call `addDocuments`
internally, might as well avoid that indirection.
---
 .../index/engine/InternalEngine.java            |  6 +-----
 .../index/engine/InternalEngineTests.java       | 17 +++++++++++++++++
 .../replication/IndexLevelReplicationTests.java | 10 ++++++++++
 .../RecoveryDuringReplicationTests.java         | 10 ++++++++++
 .../indices/recovery/RecoveryTests.java         | 10 ++++++++++
 5 files changed, 48 insertions(+), 5 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 80fed0f3092e5..58bffa0369f44 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -1470,11 +1470,7 @@ private boolean mayHaveBeenIndexedBefore(Index index) {
     }
 
     private void addDocs(final List<LuceneDocument> docs, final IndexWriter indexWriter) throws IOException {
-        if (docs.size() > 1) {
-            indexWriter.addDocuments(docs);
-        } else {
-            indexWriter.addDocument(docs.get(0));
-        }
+        indexWriter.addDocuments(docs);
         numDocAppends.inc(docs.size());
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 04dbd70835bf4..175ce085994e5 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -145,6 +145,7 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -3737,6 +3738,14 @@ public long addDocument(Iterable<? extends IndexableField> doc) throws IOExcepti
             return super.addDocument(doc);
         }
 
+        @Override
+        public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
+            @SuppressWarnings("unchecked")
+            Collection<Iterable<? extends IndexableField>> col = asInstanceOf(Collection.class, docs);
+            assertThat(col, hasSize(1));
+            return addDocument(col.iterator().next());
+        }
+
         private void maybeThrowFailure() throws IOException {
             if (failureToThrow.get() != null) {
                 Exception failure = failureToThrow.get().get();
@@ -6992,6 +7001,14 @@ public long addDocument(Iterable<? extends IndexableField> doc) throws IOExcepti
                 }
                 return super.addDocument(doc);
             }
+
+            @Override
+            public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
+                @SuppressWarnings("unchecked")
+                Collection<Iterable<? extends IndexableField>> col = asInstanceOf(Collection.class, docs);
+                assertThat(col, hasSize(1));
+                return addDocument(col.iterator().next());
+            }
         };
         try (
             Store store = createStore();
diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
index 4bde3d5688d06..1d8633db28f16 100644
--- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
@@ -47,6 +47,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -63,6 +64,7 @@
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -433,6 +435,14 @@ public long addDocument(Iterable<? extends IndexableField> doc) throws IOExcepti
                     throw indexException;
                 }
             }
+
+            @Override
+            public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
+                @SuppressWarnings("unchecked")
+                Collection<Iterable<? extends IndexableField>> col = asInstanceOf(Collection.class, docs);
+                assertThat(col, hasSize(1));
+                return addDocument(col.iterator().next());
+            }
         }, null, null, config);
         try (ReplicationGroup shards = new ReplicationGroup(buildIndexMetadata(0)) {
             @Override
diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
index 330571d53f29a..ff6b27924404e 100644
--- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
@@ -50,6 +50,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Optional;
@@ -67,6 +68,7 @@
 import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.in;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
@@ -898,6 +900,14 @@ public long addDocument(final Iterable<? extends IndexableField> doc) throws IOE
                     }
                     return super.addDocument(doc);
                 }
+
+                @Override
+                public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
+                    @SuppressWarnings("unchecked")
+                    Collection<Iterable<? extends IndexableField>> col = asInstanceOf(Collection.class, docs);
+                    assertThat(col, hasSize(1));
+                    return addDocument(col.iterator().next());
+                }
             }, null, null, config);
         }
 
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
index 9590d83c87263..d5ac683569eba 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
@@ -48,6 +48,7 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -57,6 +58,7 @@
 
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 
@@ -423,6 +425,14 @@ public long addDocument(Iterable<? extends IndexableField> doc) throws IOExcepti
                             }
                             return super.addDocument(doc);
                         }
+
+                        @Override
+                        public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
+                            @SuppressWarnings("unchecked")
+                            Collection<Iterable<? extends IndexableField>> col = asInstanceOf(Collection.class, docs);
+                            assertThat(col, hasSize(1));
+                            return addDocument(col.iterator().next());
+                        }
                     }, null, null, config);
                 }
             }

From 78d35cebd2ca5e6e181e660719b567c9a56ee48d Mon Sep 17 00:00:00 2001
From: Ievgen Degtiarenko <ievgen.degtiarenko@elastic.co>
Date: Mon, 3 Jun 2024 14:00:24 +0200
Subject: [PATCH 101/208] Unmute tests with resolved issues (#109279)

---
 muted-tests.yml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index de8481fcbb02b..d7d47a48fee23 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -1,6 +1,4 @@
 tests:
-- class: "org.elasticsearch.xpack.transform.transforms.scheduling.MonotonicClockTests"
-  issue: "https://github.com/elastic/elasticsearch/issues/108529"
 - class: "org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterTests"
   issue: "https://github.com/elastic/elasticsearch/issues/108649"
   method: "testManyRandomDocs"
@@ -22,15 +20,9 @@ tests:
 - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108857"
   method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
-- class: "org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108950"
-  method: "test {p0=health/10_basic/cluster health basic test}"
 - class: "org.elasticsearch.upgrades.SearchStatesIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108991"
   method: "testCanMatch"
-- class: "org.elasticsearch.upgrades.AggregationsIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108992"
-  method: "testHistogram"
 - class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108993"
   method: "testTrainedModelInference"

From 8baa02bc571384c7ab33c26adf6fdbeb6160f12a Mon Sep 17 00:00:00 2001
From: Pat Whelan <pat.whelan@elastic.co>
Date: Mon, 3 Jun 2024 08:29:11 -0400
Subject: [PATCH 102/208] [Transform] Prevent concurrent jobs during cleanup
 (#109047)

It is possible for two threads to be running an AsyncTwoPhaseIndexer job
at the same time.

- Thread 1 calls `finishAndSetState()`, state changes from `INDEXING` to
  `STARTED`.
- Thread 2 calls `maybeTriggerAsyncJob()`, state changes from `STARTED`
  to `INDEXING`.
- Thread 1 calls `doSaveState()` and `afterFinishOrFailure()` while
  Thread 2 calls `onStart()`.  Both share memory, so it is possible for
  them to conflict with one another.

AsyncTwoPhaseIndexer now manages a boolean as an additional control
gating `maybeTriggerAsyncJob()` called `isJobFinishing`.

- Thread 1 sets `isJobFinishing` to true, then calls
  `finishAndSetState()`.
- Thread 2 calls `maybeTriggerAsyncJob()` and exits early because
  `isJobFinishing` is true.
- Thread 1 finishes `afterFinishOrFailure()` and sets `isJobFinishing`
  to false.
- Thread 2 will retry `maybeTriggerAsyncJob()` and successfully start.

Relate #67121
---
 docs/changelog/109047.yaml                    |  5 ++
 .../core/indexing/AsyncTwoPhaseIndexer.java   | 74 ++++++++++++-------
 .../transforms/TransformIndexer.java          | 16 ----
 .../TransformIndexerStateTests.java           | 63 ++++++++++++++++
 4 files changed, 115 insertions(+), 43 deletions(-)
 create mode 100644 docs/changelog/109047.yaml

diff --git a/docs/changelog/109047.yaml b/docs/changelog/109047.yaml
new file mode 100644
index 0000000000000..85a8808353a08
--- /dev/null
+++ b/docs/changelog/109047.yaml
@@ -0,0 +1,5 @@
+pr: 109047
+summary: Prevent concurrent jobs during cleanup
+area: Transform
+type: bug
+issues: []
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
index 48f73fc352dd1..16b55c63b81da 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
@@ -57,6 +57,7 @@ public abstract class AsyncTwoPhaseIndexer<JobPosition, JobStats extends Indexer
     private final AtomicReference<JobPosition> position;
     private final ThreadPool threadPool;
     private final Object lock;
+    private final AtomicBoolean isJobFinishing;
 
     // throttling implementation
     private volatile float currentMaxDocsPerSecond;
@@ -115,6 +116,7 @@ protected AsyncTwoPhaseIndexer(
         this.position = new AtomicReference<>(initialPosition);
         this.stats = jobStats;
         this.lock = lock;
+        this.isJobFinishing = new AtomicBoolean(false);
     }
 
     /**
@@ -147,7 +149,10 @@ public JobStats getStats() {
      *         job was already aborted).
      */
     public IndexerState start() {
-        state.compareAndSet(IndexerState.STOPPED, IndexerState.STARTED);
+        if (state.compareAndSet(IndexerState.STOPPED, IndexerState.STARTED)) {
+            // in case something happens and isJobFinishing gets stuck as true, stop() and start() can reset it
+            isJobFinishing.set(false);
+        }
         return state.get();
     }
 
@@ -224,7 +229,7 @@ public boolean maybeTriggerAsyncJob(long now) {
                 case STARTED -> {
                     logger.debug("Schedule was triggered for job [" + getJobId() + "], state: [" + currentState + "]");
                     stats.incrementNumInvocations(1);
-                    if (state.compareAndSet(IndexerState.STARTED, IndexerState.INDEXING)) {
+                    if (startJob()) {
                         // fire off the search. Note this is async, the method will return from here
                         threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> {
                             onStart(now, ActionListener.wrap(r -> {
@@ -232,23 +237,13 @@ public boolean maybeTriggerAsyncJob(long now) {
                                 if (r) {
                                     nextSearch();
                                 } else {
-                                    onFinish(
-                                        ActionListener.wrap(
-                                            onFinishResponse -> doSaveState(
-                                                finishAndSetState(),
-                                                position.get(),
-                                                this::afterFinishOrFailure
-                                            ),
-                                            onFinishFailure -> doSaveState(finishAndSetState(), position.get(), this::afterFinishOrFailure)
-                                        )
-                                    );
+                                    onFinish(finishJobListener());
                                 }
                             }, this::finishWithFailure));
                         });
                         logger.debug("Beginning to index [" + getJobId() + "], state: [" + currentState + "]");
                         return true;
                     } else {
-                        logger.debug("Could not move from STARTED to INDEXING state because current state is [" + state.get() + "]");
                         return false;
                     }
                 }
@@ -260,6 +255,41 @@ public boolean maybeTriggerAsyncJob(long now) {
         }
     }
 
+    /**
+     * isJobFinishing is checked here, before moving from STARTED to INDEXING, in case the previous job is still cleaning up from its run.
+     * See {@link #finishJob()}.
+     */
+    private boolean startJob() {
+        if (isJobFinishing.get() == false && state.compareAndSet(IndexerState.STARTED, IndexerState.INDEXING)) {
+            return true;
+        } else {
+            logger.debug(
+                "Could not start job because current state is [{}] and another job may be finishing [{}]",
+                state::get,
+                isJobFinishing::get
+            );
+            return false;
+        }
+    }
+
+    /**
+     * finishAndSetState can toggle the IndexerState back to STARTED, allowing another thread to start another job.
+     * In order to give doSaveState and afterFinishOrFailure time to clean up the current job, toggle isJobFinishing around those
+     * operations. This toggle is a boolean rather than a lock so the second thread doesn't block and wait.
+     * See gh#67121
+     */
+    private void finishJob() {
+        isJobFinishing.set(true);
+        doSaveState(finishAndSetState(), position.get(), () -> {
+            afterFinishOrFailure();
+            isJobFinishing.set(false);
+        });
+    }
+
+    private <T> ActionListener<T> finishJobListener() {
+        return ActionListener.wrap(r -> finishJob(), e -> finishJob());
+    }
+
     /**
      * Checks if the state should be persisted, if true doSaveState is called before continuing. Inherited classes
      * can override this, to provide a better logic, when state should be saved.
@@ -431,7 +461,7 @@ private void finishWithIndexingFailure(Exception exc) {
 
     private void finishWithFailure(Exception exc) {
         onFailure(exc);
-        doSaveState(finishAndSetState(), position.get(), this::afterFinishOrFailure);
+        finishJob();
     }
 
     private IndexerState finishAndSetState() {
@@ -488,12 +518,7 @@ private void onSearchResponse(SearchResponse searchResponse) {
             if (searchResponse == null) {
                 logger.debug("No indexing necessary for job [{}], saving state and shutting down.", getJobId());
                 // execute finishing tasks
-                onFinish(
-                    ActionListener.wrap(
-                        r -> doSaveState(finishAndSetState(), position.get(), this::afterFinishOrFailure),
-                        e -> doSaveState(finishAndSetState(), position.get(), this::afterFinishOrFailure)
-                    )
-                );
+                onFinish(finishJobListener());
                 return;
             }
 
@@ -514,12 +539,7 @@ private void onSearchResponse(SearchResponse searchResponse) {
                 position.set(iterationResult.getPosition());
                 stats.markEndProcessing();
                 // execute finishing tasks
-                onFinish(
-                    ActionListener.wrap(
-                        r -> doSaveState(finishAndSetState(), position.get(), this::afterFinishOrFailure),
-                        e -> doSaveState(finishAndSetState(), position.get(), this::afterFinishOrFailure)
-                    )
-                );
+                onFinish(finishJobListener());
                 return;
             }
 
@@ -635,7 +655,7 @@ private boolean checkState(IndexerState currentState) {
 
             case STOPPING:
                 logger.info("Indexer job encountered [" + IndexerState.STOPPING + "] state, halting indexer.");
-                doSaveState(finishAndSetState(), getPosition(), this::afterFinishOrFailure);
+                finishJob();
                 return false;
 
             case STOPPED:
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java
index c2fb0f12dc53b..201f20ac1eb09 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java
@@ -657,22 +657,6 @@ public boolean maybeTriggerAsyncJob(long now) {
                 return false;
             }
 
-            /*
-             * ignore if indexer thread is shutting down (after finishing a checkpoint)
-             * shutting down means:
-             *  - indexer has finished a checkpoint and called onFinish
-             *  - indexer state has changed from indexing to started
-             *  - state persistence has been called but has _not_ returned yet
-             *
-             *  If we trigger the indexer in this situation the 2nd indexer thread might
-             *  try to save state at the same time, causing a version conflict
-             *  see gh#67121
-             */
-            if (indexerThreadShuttingDown) {
-                logger.debug("[{}] indexer thread is shutting down. Ignoring trigger.", getJobId());
-                return false;
-            }
-
             return super.maybeTriggerAsyncJob(now);
         }
     }
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java
index 01a2db839b7d8..e3a4ef118f611 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java
@@ -122,6 +122,7 @@ class MockedTransformIndexer extends TransformIndexer {
         private CountDownLatch searchLatch;
         private CountDownLatch doProcessLatch;
         private CountDownLatch finishLatch = new CountDownLatch(1);
+        private CountDownLatch afterFinishLatch;
 
         MockedTransformIndexer(
             ThreadPool threadPool,
@@ -307,6 +308,16 @@ public void waitUntilFinished() throws InterruptedException {
         void finishCheckpoint() {
             searchResponse = null;
         }
+
+        @Override
+        protected void afterFinishOrFailure() {
+            maybeWaitOnLatch(afterFinishLatch);
+            super.afterFinishOrFailure();
+        }
+
+        public CountDownLatch createAfterFinishLatch(int count) {
+            return afterFinishLatch = new CountDownLatch(count);
+        }
     }
 
     class MockedTransformIndexerForStatePersistenceTesting extends TransformIndexer {
@@ -959,6 +970,58 @@ public void testStopAtCheckpointForThrottledTransform() throws Exception {
         assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STOPPED)), 5, TimeUnit.SECONDS);
     }
 
+    /**
+     * Given one indexer thread is finishing its run
+     * And that thread is after finishAndSetState() but before afterFinishOrFailure()
+     * When another thread calls maybeTriggerAsyncJob
+     * Then that other thread should not start another indexer run
+     */
+    public void testRunOneJobAtATime() throws Exception {
+        var indexer = createMockIndexer(
+            createTransformConfig(),
+            new AtomicReference<>(IndexerState.STARTED),
+            null,
+            threadPool,
+            auditor,
+            null,
+            new TransformIndexerStats(),
+            new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class))
+        );
+
+        // stop the indexer thread once it kicks off
+        var startLatch = indexer.createAwaitForStartLatch(1);
+        // stop the indexer thread before afterFinishOrFailure
+        var afterFinishLatch = indexer.createAfterFinishLatch(1);
+
+        // flip IndexerState to INDEXING
+        assertEquals(IndexerState.STARTED, indexer.start());
+        assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis()));
+        assertEquals(IndexerState.INDEXING, indexer.getState());
+
+        // now let the indexer thread run
+        indexer.finishCheckpoint();
+        startLatch.countDown();
+
+        // wait until the IndexerState flips back to STARTED
+        assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STARTED)), 5, TimeUnit.SECONDS);
+
+        assertFalse(
+            "Indexer state is STARTED, but the Indexer is not finished cleaning up from the previous run.",
+            indexer.maybeTriggerAsyncJob(System.currentTimeMillis())
+        );
+
+        // let the first job finish
+        afterFinishLatch.countDown();
+        indexer.waitUntilFinished();
+
+        // we should now (eventually) be able to schedule the next job
+        assertBusy(() -> assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())), 5, TimeUnit.SECONDS);
+
+        // stop the indexer, equivalent to _stop?force=true
+        assertFalse("Transform Indexer thread should still be running", indexer.abort());
+        assertEquals(IndexerState.ABORTING, indexer.getState());
+    }
+
     private void setStopAtCheckpoint(
         TransformIndexer indexer,
         boolean shouldStopAtCheckpoint,

From 9c2368c9f10069785d73ca07096a4f659513cf80 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Mon, 3 Jun 2024 14:43:08 +0200
Subject: [PATCH 103/208] [Inference API] Typo fix (#109284)

---
 .../java/org/elasticsearch/inference/InferenceService.java    | 4 ++--
 .../services/googleaistudio/GoogleAiStudioService.java        | 2 +-
 .../xpack/inference/services/SenderServiceTests.java          | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java
index cbe05652587f5..4bfe4ae535bed 100644
--- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java
+++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java
@@ -38,7 +38,7 @@ default void init(Client client) {}
      * @param modelId               Model Id
      * @param taskType              The model task type
      * @param config                Configuration options including the secrets
-     * @param platfromArchitectures The Set of platform architectures (OS name and hardware architecture)
+     * @param platformArchitectures The Set of platform architectures (OS name and hardware architecture)
      *                              the cluster nodes and models are running on.
      * @param parsedModelListener   A listener which will handle the resulting model or failure
      */
@@ -46,7 +46,7 @@ void parseRequestConfig(
         String modelId,
         TaskType taskType,
         Map<String, Object> config,
-        Set<String> platfromArchitectures,
+        Set<String> platformArchitectures,
         ActionListener<Model> parsedModelListener
     );
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
index f990923cee922..5d2654c072a0c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
@@ -58,7 +58,7 @@ public void parseRequestConfig(
         String inferenceEntityId,
         TaskType taskType,
         Map<String, Object> config,
-        Set<String> platfromArchitectures,
+        Set<String> platformArchitectures,
         ActionListener<Model> parsedModelListener
     ) {
         try {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java
index 672f186b37ceb..ee3403492c423 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java
@@ -152,7 +152,7 @@ public void parseRequestConfig(
             String inferenceEntityId,
             TaskType taskType,
             Map<String, Object> config,
-            Set<String> platfromArchitectures,
+            Set<String> platformArchitectures,
             ActionListener<Model> parsedModelListener
         ) {
             parsedModelListener.onResponse(null);

From fb468e1d7789818f9a4966dd069d5a6751c1a485 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Mon, 3 Jun 2024 14:43:35 +0200
Subject: [PATCH 104/208] [Inference API] Reuse EmptySettings.INSTANCE instead
 of creating new object in test (#109287)

---
 .../services/cohere/completion/CohereCompletionModelTests.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModelTests.java
index 5352d8006f1e1..aac04e301ece7 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModelTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModelTests.java
@@ -40,7 +40,7 @@ public static CohereCompletionModel createModel(String url, String apiKey, @Null
             TaskType.COMPLETION,
             "service",
             new CohereCompletionServiceSettings(url, model, null),
-            new EmptyTaskSettings(),
+            EmptyTaskSettings.INSTANCE,
             new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
         );
     }

From 0b60adcff18a92bb957915e4a1fbc7bb21303276 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 3 Jun 2024 08:55:33 -0400
Subject: [PATCH 105/208] ESQL: Update docs for `@MvEvaluator` annotation
 (#109226)

This updates the docs for the `@MvEvaluator` annotation, pointing folks
away from the position-at-a-time mode if possible. It's flexible, but
the generated code does less work for you than I'd like.
---
 .../java/org/elasticsearch/compute/ann/MvEvaluator.java    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/MvEvaluator.java b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/MvEvaluator.java
index bb2cb3bf9e5fa..2f368d6256fbf 100644
--- a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/MvEvaluator.java
+++ b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/MvEvaluator.java
@@ -26,7 +26,8 @@
  * <p>
  *     Pairwise processing is <strong>generally</strong> simpler and looks
  *     like {@code int process(int current, int next)}. Use it when the result
- *     is a primitive.</p>
+ *     is a primitive.
+ * </p>
  * <p>
  *     Accumulator processing is a bit more complex and looks like
  *     {@code void process(State state, int v)} and it useful when you need to
@@ -35,7 +36,9 @@
  * <p>
  *     Position at a time processing just hands the block, start index, and end index
  *     to the processor and is useful when none of the others fit. It looks like
- *     {@code long process(LongBlock block, int start, int end)}.
+ *     {@code long process(LongBlock block, int start, int end)} and is the most
+ *     flexible, but the choice where the code generation does the least work for you.
+ *     You should only use this if pairwise and state based processing aren't options.
  * </p>
  * <p>
  *     Pairwise and accumulator processing support a {@code finish = "finish_method"}

From 4d1e27b8b32cce278e9601deb6113099f0484141 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 3 Jun 2024 09:02:36 -0400
Subject: [PATCH 106/208] ESQL: Update guide for building scalar functions
 (#109224)

This updates the guide for building your first scalar functions, mostly
to include a list of the annotations that you can add to your `process`
method.
---
 .../convert/AbstractConvertFunction.java      | 17 ++++-
 .../math/AbstractTrigonometricFunction.java   |  3 +
 .../function/scalar/package-info.java         | 75 ++++++++++++++-----
 3 files changed, 76 insertions(+), 19 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
index f04f41813d579..54d9fcb59f476 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
@@ -58,7 +58,7 @@ protected AbstractConvertFunction(Source source, Expression field) {
     /**
      * Build the evaluator given the evaluator a multivalued field.
      */
-    protected ExpressionEvaluator.Factory evaluator(ExpressionEvaluator.Factory fieldEval) {
+    protected final ExpressionEvaluator.Factory evaluator(ExpressionEvaluator.Factory fieldEval) {
         DataType sourceType = field().dataType();
         var factory = factories().get(sourceType);
         if (factory == null) {
@@ -98,6 +98,21 @@ interface BuildFactory {
         ExpressionEvaluator.Factory build(ExpressionEvaluator.Factory field, Source source);
     }
 
+    /**
+     * A map from input type to {@link ExpressionEvaluator} ctor. Usually implemented like:
+     * <pre>{@code
+     *     private static final Map<DataType, BuildFactory> EVALUATORS = Map.ofEntries(
+     *         Map.entry(BOOLEAN, (field, source) -> field),
+     *         Map.entry(KEYWORD, ToBooleanFromStringEvaluator.Factory::new),
+     *         ...
+     *     );
+     *
+     *     @Override
+     *     protected Map<DataType, BuildFactory> factories() {
+     *         return EVALUATORS;
+     *     }
+     * }</pre>
+     */
     protected abstract Map<DataType, BuildFactory> factories();
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java
index 7ec618b621c55..1643d1b21ca5d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java
@@ -28,6 +28,9 @@ abstract class AbstractTrigonometricFunction extends UnaryScalarFunction {
         super(source, field);
     }
 
+    /**
+     * Build an evaluator for this function given the evaluator for it's input.
+     */
     protected abstract EvalOperator.ExpressionEvaluator.Factory doubleEvaluator(EvalOperator.ExpressionEvaluator.Factory field);
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java
index 3f6ad8ca3ab0a..bf0e10f817e33 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java
@@ -8,7 +8,7 @@
 /**
  * Functions that take a row of data and produce a row of data without holding
  * any state between rows. This includes both the {@link org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction}
- * subclass to link into the QL infrastucture and the {@link org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator}
+ * subclass to link into the ESQL core infrastructure and the {@link org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator}
  * implementation to run the actual function.
  *
  * <h2>Guide to adding new function</h2>
@@ -47,7 +47,7 @@
  *     <li>
  *         Pick one of the csv-spec files in {@code x-pack/plugin/esql/qa/testFixtures/src/main/resources/}
  *         and add a test for the function you want to write. These files are roughly themed but there
- *         isn't a strong guiding principle in the theme.
+ *         isn't a strong guiding principle in the organization.
  *     </li>
  *     <li>
  *         Rerun the {@code CsvTests} and watch your new test fail. Yay, TDD doing it's job.
@@ -56,20 +56,51 @@
  *         Find a function in this package similar to the one you are working on and copy it to build
  *         yours. There's some ceremony required in each function class to make it constant foldable
  *         and return the right types. Take a stab at these, but don't worry too much about getting
- *         it right.
+ *         it right. Your function might extend from one of several abstract base classes, all of
+ *         those are fine for this guide, but might have special instructions called out later.
+ *         Known good base classes:
+ *         <ul>
+ *             <li>{@link org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction}</li>
+ *             <li>{@link org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.AbstractMultivalueFunction}</li>
+ *             <li>{@link org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction}
+ *                 or any subclass like {@code AbstractTrigonometricFunction}</li>
+ *             <li>{@link org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction}</li>
+ *             <li>{@link org.elasticsearch.xpack.esql.expression.function.scalar.math.DoubleConstantFunction}</li>
+ *         </ul>
  *     </li>
  *     <li>
  *         There are also methods annotated with {@link org.elasticsearch.compute.ann.Evaluator}
- *         that contain the actual inner implementation of the function. Modify those to look right
- *         and click {@code Build->Recompile 'FunctionName.java'} in IntelliJ or run the
- *         {@code CsvTests} again. This should generate an
- *         {@link org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator} implementation
- *         calling the method annotated with {@link org.elasticsearch.compute.ann.Evaluator}. Please commit the
- *         generated evaluator before submitting your PR.
- *     <li>
- *         Once your evaluator is generated you can implement
- *         {@link org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper#toEvaluator},
- *         having it return the generated evaluator.
+ *         that contain the actual inner implementation of the function. They are usually named
+ *         "process" or "processInts" or "processBar". Modify those to look right and run the {@code CsvTests}
+ *         again. This should generate an {@link org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator}
+ *         implementation calling the method annotated with {@link org.elasticsearch.compute.ann.Evaluator}.
+ *.        To make it work with IntelliJ, also click {@code Build->Recompile 'FunctionName.java'}.
+ *         Please commit the generated evaluator before submitting your PR.
+ *         <p>
+ *             NOTE: The function you copied may have a method annotated with
+ *             {@link org.elasticsearch.compute.ann.ConvertEvaluator} or
+ *             {@link org.elasticsearch.compute.ann.MvEvaluator} instead of
+ *             {@link org.elasticsearch.compute.ann.Evaluator}. Those do similar things and the
+ *             instructions should still work for you regardless. If your function contains an implementation
+ *             of {@link org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator} written by
+ *             hand then please stop and ask for help. This is not a good first function.
+ *         </p>
+ *         <p>
+ *             NOTE 2: Regardless of which annotation is on your "process" method you can learn more
+ *             about the options for generating code from the javadocs on those annotations.
+ *         </p>
+ *     <li>
+ *         Once your evaluator is generated you can have your function return it,
+ *         generally by implementing {@link org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper#toEvaluator}.
+ *         It's possible that your abstract base class implements that function and
+ *         will need you to implement something else:
+ *         <ul>
+ *             <li>{@link org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction}: {@code factories}</li>
+ *             <li>{@link org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.AbstractMultivalueFunction}:
+ *                 {@code evaluator}</li>
+ *             <li>{@code AbstractTrigonometricFunction}: {@code doubleEvaluator}</li>
+ *             <li>{@link org.elasticsearch.xpack.esql.expression.function.scalar.math.DoubleConstantFunction}: nothing!</li>
+ *         </ul>
  *     </li>
  *     <li>
  *         Add your function to {@link org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry}.
@@ -100,24 +131,32 @@
  *         {@code ./gradlew -p x-pack/plugin/esql/ test}
  *     </li>
  *     <li>
- *         Now it's time to write some docs!
- *         Generate the docs, a syntax diagram and a table with supported types by running the tests via
- *         gradle: {@code ./gradlew x-pack:plugin:esql:test}
+ *         Now it's time to generate some docs!
+ *         Actually, running the tests in the example above should have done it for you.
  *         The generated files are
- *         <ol>
+ *         <ul>
  *              <li>{@code docs/reference/esql/functions/description/myfunction.asciidoc}</li>
  *              <li>{@code docs/reference/esql/functions/examples/myfunction.asciidoc}</li>
  *              <li>{@code docs/reference/esql/functions/layout/myfunction.asciidoc}</li>
  *              <li>{@code docs/reference/esql/functions/parameters/myfunction.asciidoc}</li>
  *              <li>{@code docs/reference/esql/functions/signature/myfunction.svg}</li>
  *              <li>{@code docs/reference/esql/functions/types/myfunction.asciidoc}</li>
- *         </ol>
+ *         </ul>
  *
  *         Make sure to commit them. Add a reference to the
  *         {@code docs/reference/esql/functions/layout/myfunction.asciidoc} in the function list
  *         docs. There are plenty of examples on how
  *         to reference those files e.g. if you are writing a Math function, you will want to
  *         list it in {@code docs/reference/esql/functions/math-functions.asciidoc}.
+ *         <p>
+ *             You can generate the docs for just your function by running
+ *             {@code ./gradlew :x-pack:plugin:esql:test -Dtests.class='*SinTests'}. It's just
+ *             running your new unit test. You should see something like:
+ *         </p>
+ *         <pre>{@code
+ *              > Task :x-pack:plugin:esql:test
+ *              ESQL Docs: Only files related to [sin.asciidoc], patching them into place
+ *         }</pre>
  *     </li>
  *     <li>
  *          Build the docs by cloning the <a href="https://github.com/elastic/docs">docs repo</a>

From 1e17452995fd3887efaee4befc5c8671c36ae569 Mon Sep 17 00:00:00 2001
From: Andrei Stefan <astefan@users.noreply.github.com>
Date: Mon, 3 Jun 2024 16:28:12 +0300
Subject: [PATCH 107/208] "manual" index creation and data loading (#109290)

For "main" don't use the Lucene classes for creating and loading the
data to not trip the java security thread access permissions.
---
 .../xpack/esql/CsvTestsDataLoader.java           | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
index e9d6b1f5c6b89..a88cd508f3f4a 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.common.CheckedBiFunction;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
@@ -195,7 +196,20 @@ public static void main(String[] args) throws IOException {
 
         try (RestClient client = builder.build()) {
             loadDataSetIntoEs(client, (restClient, indexName, indexMapping, indexSettings) -> {
-                ESRestTestCase.createIndex(restClient, indexName, indexSettings, indexMapping, null);
+                // don't use ESRestTestCase methods here or, if you do, test running the main method before making the change
+                StringBuilder jsonBody = new StringBuilder("{");
+                if (indexSettings != null && indexSettings.isEmpty() == false) {
+                    jsonBody.append("\"settings\":");
+                    jsonBody.append(Strings.toString(indexSettings));
+                    jsonBody.append(",");
+                }
+                jsonBody.append("\"mappings\":");
+                jsonBody.append(indexMapping);
+                jsonBody.append("}");
+
+                Request request = new Request("PUT", "/" + indexName);
+                request.setJsonEntity(jsonBody.toString());
+                restClient.performRequest(request);
             });
         }
     }

From f8ac9ea82bc3f582bb07b288372f6eb8f5536044 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Mon, 3 Jun 2024 15:48:04 +0200
Subject: [PATCH 108/208] [Inference API] Add Google AI Studio embeddings
 support (#109243)

---
 .../org/elasticsearch/TransportVersions.java  |   1 +
 .../InferenceNamedWriteablesProvider.java     |   8 +
 .../GoogleAiStudioActionCreator.java          |   6 +
 .../GoogleAiStudioActionVisitor.java          |   2 +
 .../GoogleAiStudioEmbeddingsAction.java       |  62 ++++
 ...oogleAiStudioEmbeddingsRequestManager.java |  64 ++++
 .../GoogleAiStudioEmbeddingsRequest.java      |  81 ++++
 ...GoogleAiStudioEmbeddingsRequestEntity.java |  78 ++++
 .../googleaistudio/GoogleAiStudioUtils.java   |   2 +
 ...oogleAiStudioEmbeddingsResponseEntity.java | 110 ++++++
 .../googleaistudio/GoogleAiStudioModel.java   |   7 +
 .../googleaistudio/GoogleAiStudioService.java |  51 ++-
 .../GoogleAiStudioServiceFields.java          |  29 ++
 .../GoogleAiStudioEmbeddingsModel.java        | 128 +++++++
 ...ogleAiStudioEmbeddingsServiceSettings.java | 197 ++++++++++
 .../elasticsearch/xpack/inference/Utils.java  |   6 +
 .../GoogleAiStudioEmbeddingsActionTests.java  | 191 ++++++++++
 ...eAiStudioEmbeddingsRequestEntityTests.java | 146 ++++++++
 .../GoogleAiStudioEmbeddingsRequestTests.java | 152 ++++++++
 ...AiStudioEmbeddingsResponseEntityTests.java | 114 ++++++
 .../GoogleAiStudioServiceTests.java           | 348 +++++++++++++++++-
 .../GoogleAiStudioEmbeddingsModelTests.java   |  66 ++++
 ...iStudioEmbeddingsServiceSettingsTests.java | 114 ++++++
 23 files changed, 1958 insertions(+), 5 deletions(-)
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsAction.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioEmbeddingsRequestManager.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioEmbeddingsRequest.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioEmbeddingsRequestEntity.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceFields.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/embeddings/GoogleAiStudioEmbeddingsRequestEntityTests.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/embeddings/GoogleAiStudioEmbeddingsRequestTests.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntityTests.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModelTests.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettingsTests.java

diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index e5428497c136d..b0d14b99f15cd 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -181,6 +181,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED = def(8_672_00_0);
     public static final TransportVersion WATCHER_REQUEST_TIMEOUTS = def(8_673_00_0);
     public static final TransportVersion ML_INFERENCE_ENHANCE_DELETE_ENDPOINT = def(8_674_00_0);
+    public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_EMBEDDINGS_ADDED = def(8_675_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
index edea0104ded16..9bdc6f148917a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
@@ -45,6 +45,7 @@
 import org.elasticsearch.xpack.inference.services.elser.ElserInternalServiceSettings;
 import org.elasticsearch.xpack.inference.services.elser.ElserMlNodeTaskSettings;
 import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionServiceSettings;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsServiceSettings;
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceServiceSettings;
 import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserSecretSettings;
 import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserServiceSettings;
@@ -264,6 +265,13 @@ private static void addGoogleAiStudioNamedWritables(List<NamedWriteableRegistry.
                 GoogleAiStudioCompletionServiceSettings::new
             )
         );
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                ServiceSettings.class,
+                GoogleAiStudioEmbeddingsServiceSettings.NAME,
+                GoogleAiStudioEmbeddingsServiceSettings::new
+            )
+        );
     }
 
     private static void addInternalElserNamedWriteables(List<NamedWriteableRegistry.Entry> namedWriteables) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java
index 51a8cc7a0bd56..86154faefabc5 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionCreator.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.xpack.inference.external.http.sender.Sender;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
 import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel;
 
 import java.util.Map;
 import java.util.Objects;
@@ -31,4 +32,9 @@ public ExecutableAction create(GoogleAiStudioCompletionModel model, Map<String,
         // no overridden model as task settings are always empty for Google AI Studio completion model
         return new GoogleAiStudioCompletionAction(sender, model, serviceComponents.threadPool());
     }
+
+    @Override
+    public ExecutableAction create(GoogleAiStudioEmbeddingsModel model, Map<String, Object> taskSettings) {
+        return new GoogleAiStudioEmbeddingsAction(sender, model, serviceComponents);
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionVisitor.java
index 090d3f9a69710..2e89200cce53b 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionVisitor.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioActionVisitor.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel;
 
 import java.util.Map;
 
@@ -16,4 +17,5 @@ public interface GoogleAiStudioActionVisitor {
 
     ExecutableAction create(GoogleAiStudioCompletionModel model, Map<String, Object> taskSettings);
 
+    ExecutableAction create(GoogleAiStudioEmbeddingsModel model, Map<String, Object> taskSettings);
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsAction.java
new file mode 100644
index 0000000000000..5ce780193c789
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsAction.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.googleaistudio;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.http.sender.GoogleAiStudioEmbeddingsRequestManager;
+import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel;
+
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage;
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError;
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException;
+
+public class GoogleAiStudioEmbeddingsAction implements ExecutableAction {
+
+    private final String failedToSendRequestErrorMessage;
+
+    private final GoogleAiStudioEmbeddingsRequestManager requestManager;
+
+    private final Sender sender;
+
+    public GoogleAiStudioEmbeddingsAction(Sender sender, GoogleAiStudioEmbeddingsModel model, ServiceComponents serviceComponents) {
+        Objects.requireNonNull(serviceComponents);
+        Objects.requireNonNull(model);
+        this.sender = Objects.requireNonNull(sender);
+        this.requestManager = new GoogleAiStudioEmbeddingsRequestManager(
+            model,
+            serviceComponents.truncator(),
+            serviceComponents.threadPool()
+        );
+        this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "Google AI Studio embeddings");
+    }
+
+    @Override
+    public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener<InferenceServiceResults> listener) {
+        try {
+            ActionListener<InferenceServiceResults> wrappedListener = wrapFailuresInElasticsearchException(
+                failedToSendRequestErrorMessage,
+                listener
+            );
+
+            sender.send(requestManager, inferenceInputs, timeout, wrappedListener);
+        } catch (ElasticsearchException e) {
+            listener.onFailure(e);
+        } catch (Exception e) {
+            listener.onFailure(createInternalServerError(e, failedToSendRequestErrorMessage));
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioEmbeddingsRequestManager.java
new file mode 100644
index 0000000000000..15c2825e7d043
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioEmbeddingsRequestManager.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.common.Truncator;
+import org.elasticsearch.xpack.inference.external.googleaistudio.GoogleAiStudioResponseHandler;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
+import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioEmbeddingsRequest;
+import org.elasticsearch.xpack.inference.external.response.googleaistudio.GoogleAiStudioEmbeddingsResponseEntity;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xpack.inference.common.Truncator.truncate;
+
+public class GoogleAiStudioEmbeddingsRequestManager extends GoogleAiStudioRequestManager {
+
+    private static final Logger logger = LogManager.getLogger(GoogleAiStudioEmbeddingsRequestManager.class);
+
+    private static final ResponseHandler HANDLER = createEmbeddingsHandler();
+
+    private static ResponseHandler createEmbeddingsHandler() {
+        return new GoogleAiStudioResponseHandler("google ai studio embeddings", GoogleAiStudioEmbeddingsResponseEntity::fromResponse);
+    }
+
+    private final GoogleAiStudioEmbeddingsModel model;
+
+    private final Truncator truncator;
+
+    public GoogleAiStudioEmbeddingsRequestManager(GoogleAiStudioEmbeddingsModel model, Truncator truncator, ThreadPool threadPool) {
+        super(threadPool, model);
+        this.model = Objects.requireNonNull(model);
+        this.truncator = Objects.requireNonNull(truncator);
+    }
+
+    @Override
+    public Runnable create(
+        String query,
+        List<String> input,
+        RequestSender requestSender,
+        Supplier<Boolean> hasRequestCompletedFunction,
+        HttpClientContext context,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        var truncatedInput = truncate(input, model.getServiceSettings().maxInputTokens());
+        GoogleAiStudioEmbeddingsRequest request = new GoogleAiStudioEmbeddingsRequest(truncator, truncatedInput, model);
+
+        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioEmbeddingsRequest.java
new file mode 100644
index 0000000000000..a96cbf2afb27a
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioEmbeddingsRequest.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.googleaistudio;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.common.Truncator;
+import org.elasticsearch.xpack.inference.external.request.HttpRequest;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+public class GoogleAiStudioEmbeddingsRequest implements GoogleAiStudioRequest {
+
+    private final Truncator truncator;
+
+    private final Truncator.TruncationResult truncationResult;
+
+    private final GoogleAiStudioEmbeddingsModel model;
+
+    public GoogleAiStudioEmbeddingsRequest(Truncator truncator, Truncator.TruncationResult input, GoogleAiStudioEmbeddingsModel model) {
+        this.truncator = Objects.requireNonNull(truncator);
+        this.truncationResult = Objects.requireNonNull(input);
+        this.model = Objects.requireNonNull(model);
+    }
+
+    @Override
+    public HttpRequest createHttpRequest() {
+        HttpPost httpPost = new HttpPost(model.uri());
+
+        ByteArrayEntity byteEntity = new ByteArrayEntity(
+            Strings.toString(
+                new GoogleAiStudioEmbeddingsRequestEntity(
+                    truncationResult.input(),
+                    model.getServiceSettings().modelId(),
+                    model.getServiceSettings().dimensions()
+                )
+            ).getBytes(StandardCharsets.UTF_8)
+        );
+
+        httpPost.setEntity(byteEntity);
+        httpPost.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType());
+
+        GoogleAiStudioRequest.decorateWithApiKeyParameter(httpPost, model.getSecretSettings());
+
+        return new HttpRequest(httpPost, getInferenceEntityId());
+    }
+
+    @Override
+    public String getInferenceEntityId() {
+        return model.getInferenceEntityId();
+    }
+
+    @Override
+    public URI getURI() {
+        return model.uri();
+    }
+
+    @Override
+    public Request truncate() {
+        var truncatedInput = truncator.truncate(truncationResult.input());
+
+        return new GoogleAiStudioEmbeddingsRequest(truncator, truncatedInput, model);
+    }
+
+    @Override
+    public boolean[] getTruncationInfo() {
+        return truncationResult.truncated().clone();
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioEmbeddingsRequestEntity.java
new file mode 100644
index 0000000000000..9d40f1cf097ec
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioEmbeddingsRequestEntity.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.googleaistudio;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.core.Strings.format;
+
+public record GoogleAiStudioEmbeddingsRequestEntity(List<String> inputs, String model, @Nullable Integer dimensions)
+    implements
+        ToXContentObject {
+
+    private static final String REQUESTS_FIELD = "requests";
+    private static final String MODEL_FIELD = "model";
+
+    private static final String MODELS_PREFIX = "models";
+    private static final String CONTENT_FIELD = "content";
+    private static final String PARTS_FIELD = "parts";
+    private static final String TEXT_FIELD = "text";
+
+    private static final String OUTPUT_DIMENSIONALITY_FIELD = "outputDimensionality";
+
+    public GoogleAiStudioEmbeddingsRequestEntity {
+        Objects.requireNonNull(inputs);
+        Objects.requireNonNull(model);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.startArray(REQUESTS_FIELD);
+
+        for (String input : inputs) {
+            builder.startObject();
+            builder.field(MODEL_FIELD, format("%s/%s", MODELS_PREFIX, model));
+
+            {
+                builder.startObject(CONTENT_FIELD);
+
+                {
+                    builder.startArray(PARTS_FIELD);
+
+                    {
+                        builder.startObject();
+                        builder.field(TEXT_FIELD, input);
+                        builder.endObject();
+                    }
+
+                    builder.endArray();
+                }
+
+                builder.endObject();
+            }
+
+            if (dimensions != null) {
+                builder.field(OUTPUT_DIMENSIONALITY_FIELD, dimensions);
+            }
+
+            builder.endObject();
+        }
+
+        builder.endArray();
+        builder.endObject();
+
+        return builder;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java
index d63a0bbe2af91..81ad5b6203682 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioUtils.java
@@ -17,6 +17,8 @@ public class GoogleAiStudioUtils {
 
     public static final String GENERATE_CONTENT_ACTION = "generateContent";
 
+    public static final String BATCH_EMBED_CONTENTS_ACTION = "batchEmbedContents";
+
     private GoogleAiStudioUtils() {}
 
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java
new file mode 100644
index 0000000000000..204738f2a2552
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.googleaistudio;
+
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.consumeUntilObjectEnd;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
+import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
+
+public class GoogleAiStudioEmbeddingsResponseEntity {
+
+    private static final String FAILED_TO_FIND_FIELD_TEMPLATE =
+        "Failed to find required field [%s] in Google AI Studio embeddings response";
+
+    /**
+     * Parses the Google AI Studio batch embeddings response (will be used for single and batch embeddings).
+     * For a request like:
+     *
+     * <pre>
+     *     <code>
+     *         {
+     *             "inputs": ["Embed this", "Embed this, too"]
+     *         }
+     *     </code>
+     * </pre>
+     *
+     * The response would look like:
+     *
+     * <pre>
+     *     <code>
+     *  {
+     *     "embeddings": [
+     *         {
+     *             "values": [
+     *                 -0.00606332,
+     *                 0.058092743,
+     *                 -0.06390548
+     *             ]
+     *         },
+     *         {
+     *             "values": [
+     *               -0.00606332,
+     *               -0.06390548,
+     *                0.058092743
+     *             ]
+     *         }
+     *     ]
+     *  }
+     *
+     *     </code>
+     * </pre>
+     */
+
+    public static TextEmbeddingResults fromResponse(Request request, HttpResult response) throws IOException {
+        var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
+
+        try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) {
+            moveToFirstToken(jsonParser);
+
+            XContentParser.Token token = jsonParser.currentToken();
+            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+
+            positionParserAtTokenAfterField(jsonParser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+            List<TextEmbeddingResults.Embedding> embeddingList = XContentParserUtils.parseList(
+                jsonParser,
+                GoogleAiStudioEmbeddingsResponseEntity::parseEmbeddingObject
+            );
+
+            return new TextEmbeddingResults(embeddingList);
+        }
+    }
+
+    private static TextEmbeddingResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+
+        positionParserAtTokenAfterField(parser, "values", FAILED_TO_FIND_FIELD_TEMPLATE);
+
+        List<Float> embeddingValuesList = XContentParserUtils.parseList(parser, GoogleAiStudioEmbeddingsResponseEntity::parseEmbeddingList);
+        // parse and discard the rest of the object
+        consumeUntilObjectEnd(parser);
+
+        return TextEmbeddingResults.Embedding.of(embeddingValuesList);
+    }
+
+    private static float parseEmbeddingList(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
+        return parser.floatValue();
+    }
+
+    private GoogleAiStudioEmbeddingsResponseEntity() {}
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java
index 4ddffd0bae615..d817a3bbb73ef 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioModel.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor;
 
@@ -31,6 +32,12 @@ public GoogleAiStudioModel(
         this.rateLimitServiceSettings = Objects.requireNonNull(rateLimitServiceSettings);
     }
 
+    public GoogleAiStudioModel(GoogleAiStudioModel model, ServiceSettings serviceSettings) {
+        super(model, serviceSettings);
+
+        rateLimitServiceSettings = model.rateLimitServiceSettings();
+    }
+
     public abstract ExecutableAction accept(GoogleAiStudioActionVisitor creator, Map<String, Object> taskSettings, InputType inputType);
 
     public GoogleAiStudioRateLimitServiceSettings rateLimitServiceSettings() {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
index 5d2654c072a0c..f8720448b0f4f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
@@ -20,15 +20,20 @@
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.SimilarityMeasure;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.inference.common.EmbeddingRequestChunker;
 import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionCreator;
 import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.SenderService;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.ServiceUtils;
 import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsServiceSettings;
 
 import java.util.List;
 import java.util.Map;
@@ -39,6 +44,7 @@
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
+import static org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioServiceFields.EMBEDDING_MAX_BATCH_SIZE;
 
 public class GoogleAiStudioService extends SenderService {
 
@@ -104,6 +110,14 @@ private static GoogleAiStudioModel createModel(
                 taskSettings,
                 secretSettings
             );
+            case TEXT_EMBEDDING -> new GoogleAiStudioEmbeddingsModel(
+                inferenceEntityId,
+                taskType,
+                NAME,
+                serviceSettings,
+                taskSettings,
+                secretSettings
+            );
             default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
         };
     }
@@ -168,6 +182,34 @@ public TransportVersion getMinimalSupportedVersion() {
         return TransportVersions.ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED;
     }
 
+    @Override
+    public void checkModelConfig(Model model, ActionListener<Model> listener) {
+        if (model instanceof GoogleAiStudioEmbeddingsModel embeddingsModel) {
+            ServiceUtils.getEmbeddingSize(
+                model,
+                this,
+                listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size)))
+            );
+        } else {
+            listener.onResponse(model);
+        }
+    }
+
+    private GoogleAiStudioEmbeddingsModel updateModelWithEmbeddingDetails(GoogleAiStudioEmbeddingsModel model, int embeddingSize) {
+        var similarityFromModel = model.getServiceSettings().similarity();
+        var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel;
+
+        GoogleAiStudioEmbeddingsServiceSettings serviceSettings = new GoogleAiStudioEmbeddingsServiceSettings(
+            model.getServiceSettings().modelId(),
+            model.getServiceSettings().maxInputTokens(),
+            embeddingSize,
+            similarityToUse,
+            model.getServiceSettings().rateLimitSettings()
+        );
+
+        return new GoogleAiStudioEmbeddingsModel(model, serviceSettings);
+    }
+
     @Override
     protected void doInfer(
         Model model,
@@ -213,6 +255,13 @@ protected void doChunkedInfer(
         TimeValue timeout,
         ActionListener<List<ChunkedInferenceServiceResults>> listener
     ) {
-        throw new UnsupportedOperationException("Chunked inference not supported yet for Google AI Studio");
+        GoogleAiStudioModel googleAiStudioModel = (GoogleAiStudioModel) model;
+        var actionCreator = new GoogleAiStudioActionCreator(getSender(), getServiceComponents());
+
+        var batchedRequests = new EmbeddingRequestChunker(input, EMBEDDING_MAX_BATCH_SIZE).batchRequestsWithListeners(listener);
+        for (var request : batchedRequests) {
+            var action = googleAiStudioModel.accept(actionCreator, taskSettings, inputType);
+            action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener());
+        }
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceFields.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceFields.java
new file mode 100644
index 0000000000000..72471251fd86c
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceFields.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.googleaistudio;
+
+public class GoogleAiStudioServiceFields {
+
+    /**
+     * Didn't find any documentation on this, but provoked it through a large enough request, which returned:
+     *
+     * <pre>
+     *     <code>
+     *         {
+     *             "error": {
+     *                  "code": 400,
+    *                      "message": "* BatchEmbedContentsRequest.requests: at most 100 requests can be in one batch\n",
+     *                   "status": "INVALID_ARGUMENT"
+     *              }
+     *          }
+     *     </code>
+     * </pre>
+     */
+    static final int EMBEDDING_MAX_BATCH_SIZE = 100;
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
new file mode 100644
index 0000000000000..0794e47d338bc
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.googleaistudio.embeddings;
+
+import org.apache.http.client.utils.URIBuilder;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.EmptyTaskSettings;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor;
+import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioUtils;
+import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioModel;
+import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioSecretSettings;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+
+import static org.elasticsearch.core.Strings.format;
+
+public class GoogleAiStudioEmbeddingsModel extends GoogleAiStudioModel {
+
+    private URI uri;
+
+    public GoogleAiStudioEmbeddingsModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        String service,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        Map<String, Object> secrets
+    ) {
+        this(
+            inferenceEntityId,
+            taskType,
+            service,
+            GoogleAiStudioEmbeddingsServiceSettings.fromMap(serviceSettings),
+            EmptyTaskSettings.INSTANCE,
+            GoogleAiStudioSecretSettings.fromMap(secrets)
+        );
+    }
+
+    public GoogleAiStudioEmbeddingsModel(GoogleAiStudioEmbeddingsModel model, GoogleAiStudioEmbeddingsServiceSettings serviceSettings) {
+        super(model, serviceSettings);
+    }
+
+    // Should only be used directly for testing
+    GoogleAiStudioEmbeddingsModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        String service,
+        GoogleAiStudioEmbeddingsServiceSettings serviceSettings,
+        TaskSettings taskSettings,
+        @Nullable GoogleAiStudioSecretSettings secrets
+    ) {
+        super(
+            new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings),
+            new ModelSecrets(secrets),
+            serviceSettings
+        );
+        try {
+            this.uri = buildUri(serviceSettings.modelId());
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    // Should only be used directly for testing
+    GoogleAiStudioEmbeddingsModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        String service,
+        String uri,
+        GoogleAiStudioEmbeddingsServiceSettings serviceSettings,
+        TaskSettings taskSettings,
+        @Nullable GoogleAiStudioSecretSettings secrets
+    ) {
+        super(
+            new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings),
+            new ModelSecrets(secrets),
+            serviceSettings
+        );
+        try {
+            this.uri = new URI(uri);
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public GoogleAiStudioEmbeddingsServiceSettings getServiceSettings() {
+        return (GoogleAiStudioEmbeddingsServiceSettings) super.getServiceSettings();
+    }
+
+    @Override
+    public GoogleAiStudioSecretSettings getSecretSettings() {
+        return (GoogleAiStudioSecretSettings) super.getSecretSettings();
+    }
+
+    public URI uri() {
+        return uri;
+    }
+
+    @Override
+    public ExecutableAction accept(GoogleAiStudioActionVisitor visitor, Map<String, Object> taskSettings, InputType inputType) {
+        return visitor.create(this, taskSettings);
+    }
+
+    public static URI buildUri(String model) throws URISyntaxException {
+        return new URIBuilder().setScheme("https")
+            .setHost(GoogleAiStudioUtils.HOST_SUFFIX)
+            .setPathSegments(
+                GoogleAiStudioUtils.V1,
+                GoogleAiStudioUtils.MODELS,
+                format("%s:%s", model, GoogleAiStudioUtils.BATCH_EMBED_CONTENTS_ACTION)
+            )
+            .build();
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java
new file mode 100644
index 0000000000000..07d07dc533f06
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java
@@ -0,0 +1,197 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.googleaistudio.embeddings;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity;
+
+public class GoogleAiStudioEmbeddingsServiceSettings extends FilteredXContentObject
+    implements
+        ServiceSettings,
+        GoogleAiStudioRateLimitServiceSettings {
+
+    public static final String NAME = "google_ai_studio_embeddings_service_settings";
+
+    /**
+     * Rate limits are defined at <a href="https://ai.google.dev/pricing">Google Gemini API Pricing</a>.
+     * For pay-as-you-go you've 360 requests per minute.
+     */
+    private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(360);
+
+    public static GoogleAiStudioEmbeddingsServiceSettings fromMap(Map<String, Object> map) {
+        ValidationException validationException = new ValidationException();
+
+        String model = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        Integer maxInputTokens = extractOptionalPositiveInteger(
+            map,
+            MAX_INPUT_TOKENS,
+            ModelConfigurations.SERVICE_SETTINGS,
+            validationException
+        );
+        SimilarityMeasure similarityMeasure = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        Integer dims = extractOptionalPositiveInteger(map, DIMENSIONS, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new GoogleAiStudioEmbeddingsServiceSettings(model, maxInputTokens, dims, similarityMeasure, rateLimitSettings);
+    }
+
+    private final String modelId;
+
+    private final RateLimitSettings rateLimitSettings;
+
+    private final Integer dims;
+
+    private final Integer maxInputTokens;
+
+    private final SimilarityMeasure similarity;
+
+    public GoogleAiStudioEmbeddingsServiceSettings(
+        String modelId,
+        @Nullable Integer maxInputTokens,
+        @Nullable Integer dims,
+        @Nullable SimilarityMeasure similarity,
+        @Nullable RateLimitSettings rateLimitSettings
+    ) {
+        this.modelId = modelId;
+        this.maxInputTokens = maxInputTokens;
+        this.dims = dims;
+        this.similarity = similarity;
+        this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS);
+    }
+
+    public GoogleAiStudioEmbeddingsServiceSettings(StreamInput in) throws IOException {
+        this.modelId = in.readString();
+        this.maxInputTokens = in.readOptionalVInt();
+        this.dims = in.readOptionalVInt();
+        this.similarity = in.readOptionalEnum(SimilarityMeasure.class);
+        this.rateLimitSettings = new RateLimitSettings(in);
+    }
+
+    @Override
+    public String modelId() {
+        return modelId;
+    }
+
+    @Override
+    public RateLimitSettings rateLimitSettings() {
+        return rateLimitSettings;
+    }
+
+    public Integer maxInputTokens() {
+        return maxInputTokens;
+    }
+
+    @Override
+    public Integer dimensions() {
+        return dims;
+    }
+
+    @Override
+    public SimilarityMeasure similarity() {
+        return similarity;
+    }
+
+    @Override
+    public DenseVectorFieldMapper.ElementType elementType() {
+        return DenseVectorFieldMapper.ElementType.FLOAT;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        toXContentFragmentOfExposedFields(builder, params);
+        rateLimitSettings.toXContent(builder, params);
+
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ML_INFERENCE_GOOGLE_AI_STUDIO_EMBEDDINGS_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(modelId);
+        out.writeOptionalVInt(maxInputTokens);
+        out.writeOptionalVInt(dims);
+        out.writeOptionalEnum(similarity);
+        rateLimitSettings.writeTo(out);
+    }
+
+    @Override
+    protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException {
+        builder.field(MODEL_ID, modelId);
+
+        if (maxInputTokens != null) {
+            builder.field(MAX_INPUT_TOKENS, maxInputTokens);
+        }
+
+        if (dims != null) {
+            builder.field(DIMENSIONS, dims);
+        }
+
+        if (similarity != null) {
+            builder.field(SIMILARITY, similarity);
+        }
+
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) return true;
+        if (object == null || getClass() != object.getClass()) return false;
+        GoogleAiStudioEmbeddingsServiceSettings that = (GoogleAiStudioEmbeddingsServiceSettings) object;
+        return Objects.equals(modelId, that.modelId)
+            && Objects.equals(rateLimitSettings, that.rateLimitSettings)
+            && Objects.equals(dims, that.dims)
+            && Objects.equals(maxInputTokens, that.maxInputTokens)
+            && similarity == that.similarity;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(modelId, rateLimitSettings, dims, maxInputTokens, similarity);
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java
index e6cfd565c2a17..58603526a9c56 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java
@@ -36,6 +36,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -153,4 +154,9 @@ public static Model getInvalidModel(String inferenceEntityId, String serviceName
 
         return mockModel;
     }
+
+    public static SimilarityMeasure randomSimilarityMeasure() {
+        return randomFrom(SimilarityMeasure.values());
+    }
+
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java
new file mode 100644
index 0000000000000..a55b3c5f5030c
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java
@@ -0,0 +1,191 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.googleaistudio;
+
+import org.apache.http.HttpHeaders;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.action.InferenceAction;
+import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
+import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool;
+import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty;
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
+import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat;
+import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
+import static org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModelTests.createModel;
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+
+public class GoogleAiStudioEmbeddingsActionTests extends ESTestCase {
+
+    private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS);
+    private final MockWebServer webServer = new MockWebServer();
+    private ThreadPool threadPool;
+    private HttpClientManager clientManager;
+
+    @Before
+    public void init() throws Exception {
+        webServer.start();
+        threadPool = createThreadPool(inferenceUtilityPool());
+        clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class));
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        clientManager.close();
+        terminate(threadPool);
+        webServer.close();
+    }
+
+    public void testExecute_ReturnsSuccessfulResponse() throws IOException {
+        var apiKey = "apiKey";
+        var model = "model";
+        var input = "input";
+        var senderFactory = new HttpRequestSender.Factory(createWithEmptySettings(threadPool), clientManager, mockClusterServiceEmpty());
+
+        try (var sender = senderFactory.createSender("test_service")) {
+            sender.start();
+
+            String responseJson = """
+                {
+                    "embeddings": [
+                        {
+                            "values": [
+                                0.0123,
+                                -0.0123
+                            ]
+                        }
+                    ]
+                }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var action = createAction(getUrl(webServer), apiKey, model, sender);
+
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            action.execute(new DocumentsOnlyInput(List.of(input)), InferenceAction.Request.DEFAULT_TIMEOUT, listener);
+
+            var result = listener.actionGet(TIMEOUT);
+
+            assertThat(result.asMap(), is(buildExpectationFloat(List.of(new float[] { 0.0123F, -0.0123F }))));
+            assertThat(webServer.requests(), hasSize(1));
+            assertThat(webServer.requests().get(0).getUri().getQuery(), endsWith(apiKey));
+            assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType()));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            assertThat(requestMap, aMapWithSize(1));
+            assertThat(
+                requestMap.get("requests"),
+                is(
+                    List.of(
+                        Map.of(
+                            "model",
+                            Strings.format("%s/%s", "models", model),
+                            "content",
+                            Map.of("parts", List.of(Map.of("text", input)))
+                        )
+                    )
+                )
+            );
+        }
+    }
+
+    public void testExecute_ThrowsElasticsearchException() {
+        var sender = mock(Sender.class);
+        doThrow(new ElasticsearchException("failed")).when(sender).send(any(), any(), any(), any());
+
+        var action = createAction(getUrl(webServer), "api_key", "model", sender);
+
+        PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+        action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener);
+
+        var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT));
+
+        assertThat(thrownException.getMessage(), is("failed"));
+    }
+
+    public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled() {
+        var sender = mock(Sender.class);
+
+        doAnswer(invocation -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<InferenceServiceResults> listener = (ActionListener<InferenceServiceResults>) invocation.getArguments()[2];
+            listener.onFailure(new IllegalStateException("failed"));
+
+            return Void.TYPE;
+        }).when(sender).send(any(), any(), any(), any());
+
+        var action = createAction(getUrl(webServer), "api_key", "model", sender);
+
+        PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+        action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener);
+
+        var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT));
+
+        assertThat(
+            thrownException.getMessage(),
+            is(format("Failed to send Google AI Studio embeddings request to [%s]", getUrl(webServer)))
+        );
+    }
+
+    public void testExecute_ThrowsException() {
+        var sender = mock(Sender.class);
+        doThrow(new IllegalArgumentException("failed")).when(sender).send(any(), any(), any(), any());
+
+        var action = createAction(getUrl(webServer), "api_key", "model", sender);
+
+        PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+        action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener);
+
+        var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT));
+
+        assertThat(
+            thrownException.getMessage(),
+            is(format("Failed to send Google AI Studio embeddings request to [%s]", getUrl(webServer)))
+        );
+    }
+
+    private GoogleAiStudioEmbeddingsAction createAction(String url, String apiKey, String modelName, Sender sender) {
+        var model = createModel(modelName, apiKey, url);
+
+        return new GoogleAiStudioEmbeddingsAction(sender, model, createWithEmptySettings(threadPool));
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/embeddings/GoogleAiStudioEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/embeddings/GoogleAiStudioEmbeddingsRequestEntityTests.java
new file mode 100644
index 0000000000000..4c3b33e1dc950
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/embeddings/GoogleAiStudioEmbeddingsRequestEntityTests.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.googleaistudio.embeddings;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioEmbeddingsRequestEntity;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString;
+
+public class GoogleAiStudioEmbeddingsRequestEntityTests extends ESTestCase {
+
+    public void testXContent_SingleRequest_WritesDimensionsIfDefined() throws IOException {
+        var entity = new GoogleAiStudioEmbeddingsRequestEntity(List.of("abc"), "model", 8);
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "requests": [
+                    {
+                        "model": "models/model",
+                        "content": {
+                            "parts": [
+                                {
+                                    "text": "abc"
+                                }
+                            ]
+                        },
+                        "outputDimensionality": 8
+                    }
+                ]
+            }
+            """));
+    }
+
+    public void testXContent_SingleRequest_DoesNotWriteDimensionsIfNull() throws IOException {
+        var entity = new GoogleAiStudioEmbeddingsRequestEntity(List.of("abc"), "model", null);
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "requests": [
+                    {
+                        "model": "models/model",
+                        "content": {
+                            "parts": [
+                                {
+                                    "text": "abc"
+                                }
+                            ]
+                        }
+                    }
+                ]
+            }
+            """));
+    }
+
+    public void testXContent_MultipleRequests_WritesDimensionsIfDefined() throws IOException {
+        var entity = new GoogleAiStudioEmbeddingsRequestEntity(List.of("abc", "def"), "model", 8);
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "requests": [
+                    {
+                        "model": "models/model",
+                        "content": {
+                            "parts": [
+                                {
+                                    "text": "abc"
+                                }
+                            ]
+                        },
+                        "outputDimensionality": 8
+                    },
+                    {
+                        "model": "models/model",
+                        "content": {
+                            "parts": [
+                                {
+                                    "text": "def"
+                                }
+                            ]
+                        },
+                        "outputDimensionality": 8
+                    }
+                ]
+            }
+            """));
+    }
+
+    public void testXContent_MultipleRequests_DoesNotWriteDimensionsIfNull() throws IOException {
+        var entity = new GoogleAiStudioEmbeddingsRequestEntity(List.of("abc", "def"), "model", null);
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "requests": [
+                    {
+                        "model": "models/model",
+                        "content": {
+                            "parts": [
+                                {
+                                    "text": "abc"
+                                }
+                            ]
+                        }
+                    },
+                    {
+                        "model": "models/model",
+                        "content": {
+                            "parts": [
+                                {
+                                    "text": "def"
+                                }
+                            ]
+                        }
+                    }
+                ]
+            }
+            """));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/embeddings/GoogleAiStudioEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/embeddings/GoogleAiStudioEmbeddingsRequestTests.java
new file mode 100644
index 0000000000000..9ce254bd8e3da
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/embeddings/GoogleAiStudioEmbeddingsRequestTests.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.googleaistudio.embeddings;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.common.Truncator;
+import org.elasticsearch.xpack.inference.common.TruncatorTests;
+import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioEmbeddingsRequest;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModelTests;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+public class GoogleAiStudioEmbeddingsRequestTests extends ESTestCase {
+
+    public void testCreateRequest_WithoutDimensionsSet() throws IOException {
+        var model = "model";
+        var apiKey = "api_key";
+        var input = "input";
+
+        var request = createRequest(model, apiKey, input, null, null);
+        var httpRequest = request.createHttpRequest();
+
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        assertThat(httpPost.getURI().toString(), endsWith(Strings.format("%s=%s", "key", apiKey)));
+        assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        assertThat(requestMap, aMapWithSize(1));
+        assertThat(
+            requestMap.get("requests"),
+            is(
+                List.of(
+                    Map.of("model", Strings.format("%s/%s", "models", model), "content", Map.of("parts", List.of(Map.of("text", input))))
+                )
+            )
+        );
+    }
+
+    public void testCreateRequest_WithDimensionsSet() throws IOException {
+        var model = "model";
+        var apiKey = "api_key";
+        var input = "input";
+        var dimensions = 8;
+
+        var request = createRequest(model, apiKey, input, null, dimensions);
+        var httpRequest = request.createHttpRequest();
+
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        assertThat(httpPost.getURI().toString(), endsWith(Strings.format("%s=%s", "key", apiKey)));
+        assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        assertThat(requestMap, aMapWithSize(1));
+        assertThat(
+            requestMap.get("requests"),
+            is(
+                List.of(
+                    Map.of(
+                        "model",
+                        Strings.format("%s/%s", "models", model),
+                        "content",
+                        Map.of("parts", List.of(Map.of("text", input))),
+                        "outputDimensionality",
+                        dimensions
+                    )
+                )
+            )
+        );
+    }
+
+    public void testTruncate_ReducesInputTextSizeByHalf() throws IOException {
+        var model = "model";
+        var apiKey = "api_key";
+        var input = "abcd";
+        var dimensions = 8;
+
+        var request = createRequest(model, apiKey, input, null, dimensions);
+        var truncatedRequest = request.truncate();
+        var httpRequest = truncatedRequest.createHttpRequest();
+
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+
+        assertThat(httpPost.getURI().toString(), endsWith(Strings.format("%s=%s", "key", apiKey)));
+        assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        assertThat(requestMap, aMapWithSize(1));
+        assertThat(
+            requestMap.get("requests"),
+            is(
+                List.of(
+                    Map.of(
+                        "model",
+                        Strings.format("%s/%s", "models", model),
+                        "content",
+                        // "abcd" reduced by half -> "ab"
+                        Map.of("parts", List.of(Map.of("text", "ab"))),
+                        "outputDimensionality",
+                        dimensions
+                    )
+                )
+            )
+        );
+    }
+
+    public void testIsTruncated_ReturnsTrue() {
+        var request = createRequest("model", "api key", "input", null, null);
+        assertFalse(request.getTruncationInfo()[0]);
+
+        var truncatedRequest = request.truncate();
+        assertTrue(truncatedRequest.getTruncationInfo()[0]);
+    }
+
+    public static GoogleAiStudioEmbeddingsRequest createRequest(
+        String model,
+        String apiKey,
+        String input,
+        @Nullable Integer maxTokens,
+        @Nullable Integer dimensions
+    ) {
+        var embeddingsModel = GoogleAiStudioEmbeddingsModelTests.createModel(model, apiKey, maxTokens, dimensions);
+
+        return new GoogleAiStudioEmbeddingsRequest(
+            TruncatorTests.createTruncator(),
+            new Truncator.TruncationResult(List.of(input), new boolean[] { false }),
+            embeddingsModel
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntityTests.java
new file mode 100644
index 0000000000000..5d5096d0b1b51
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntityTests.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.googleaistudio;
+
+import org.apache.http.HttpResponse;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+public class GoogleAiStudioEmbeddingsResponseEntityTests extends ESTestCase {
+
+    public void testFromResponse_CreatesResultsForASingleItem() throws IOException {
+        String responseJson = """
+            {
+                "embeddings": [
+                    {
+                        "values": [
+                            -0.00606332,
+                            0.058092743
+                        ]
+                    }
+                ]
+            }
+            """;
+
+        TextEmbeddingResults parsedResults = GoogleAiStudioEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(parsedResults.embeddings(), is(List.of(TextEmbeddingResults.Embedding.of(List.of(-0.00606332F, 0.058092743F)))));
+    }
+
+    public void testFromResponse_CreatesResultsForMultipleItems() throws IOException {
+        String responseJson = """
+            {
+                "embeddings": [
+                    {
+                        "values": [
+                            -0.00606332,
+                            0.058092743
+                        ]
+                    },
+                    {
+                        "values": [
+                            0.030681048,
+                            0.01714732
+                        ]
+                    }
+                ]
+            }
+            """;
+
+        TextEmbeddingResults parsedResults = GoogleAiStudioEmbeddingsResponseEntity.fromResponse(
+            mock(Request.class),
+            new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+        );
+
+        assertThat(
+            parsedResults.embeddings(),
+            is(
+                List.of(
+                    TextEmbeddingResults.Embedding.of(List.of(-0.00606332F, 0.058092743F)),
+                    TextEmbeddingResults.Embedding.of(List.of(0.030681048F, 0.01714732F))
+                )
+            )
+        );
+    }
+
+    public void testFromResponse_FailsWhenEmbeddingsFieldIsNotPresent() {
+        String responseJson = """
+            {
+                "not_embeddings": [
+                    {
+                        "values": [
+                            -0.00606332,
+                            0.058092743
+                        ]
+                    },
+                    {
+                        "values": [
+                            0.030681048,
+                            0.01714732
+                        ]
+                    }
+                ]
+            }
+            """;
+
+        var thrownException = expectThrows(
+            IllegalStateException.class,
+            () -> GoogleAiStudioEmbeddingsResponseEntity.fromResponse(
+                mock(Request.class),
+                new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8))
+            )
+        );
+
+        assertThat(thrownException.getMessage(), is("Failed to find required field [embeddings] in Google AI Studio embeddings response"));
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java
index f157622ea7291..32e912ff8529a 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java
@@ -13,13 +13,17 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.ChunkedInferenceServiceResults;
+import org.elasticsearch.inference.ChunkingOptions;
 import org.elasticsearch.inference.EmptyTaskSettings;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.inference.InputType;
 import org.elasticsearch.inference.Model;
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.SimilarityMeasure;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.http.MockResponse;
@@ -28,6 +32,7 @@
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults;
+import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingFloatResults;
 import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests;
@@ -35,12 +40,16 @@
 import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
 import org.elasticsearch.xpack.inference.services.ServiceFields;
 import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel;
+import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModelTests;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModelTests;
 import org.hamcrest.MatcherAssert;
 import org.hamcrest.Matchers;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -53,14 +62,16 @@
 import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty;
 import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
+import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat;
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
 import static org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty;
-import static org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModelTests.createModel;
 import static org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests.getSecretSettingsMap;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.hasSize;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
@@ -118,6 +129,33 @@ public void testParseRequestConfig_CreatesAGoogleAiStudioCompletionModel() throw
         }
     }
 
+    public void testParseRequestConfig_CreatesAGoogleAiStudioEmbeddingsModel() throws IOException {
+        var apiKey = "apiKey";
+        var modelId = "model";
+
+        try (var service = createGoogleAiStudioService()) {
+            ActionListener<Model> modelListener = ActionListener.wrap(model -> {
+                assertThat(model, instanceOf(GoogleAiStudioEmbeddingsModel.class));
+
+                var embeddingsModel = (GoogleAiStudioEmbeddingsModel) model;
+                assertThat(embeddingsModel.getServiceSettings().modelId(), is(modelId));
+                assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is(apiKey));
+            }, e -> fail("Model parsing should have succeeded, but failed: " + e.getMessage()));
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                getRequestConfigMap(
+                    new HashMap<>(Map.of(ServiceFields.MODEL_ID, modelId)),
+                    new HashMap<>(Map.of()),
+                    getSecretSettingsMap(apiKey)
+                ),
+                Set.of(),
+                modelListener
+            );
+        }
+    }
+
     public void testParseRequestConfig_ThrowsUnsupportedModelType() throws IOException {
         try (var service = createGoogleAiStudioService()) {
             var failureListener = getModelListenerForException(
@@ -236,6 +274,33 @@ public void testParsePersistedConfigWithSecrets_CreatesAGoogleAiStudioCompletion
         }
     }
 
+    public void testParsePersistedConfigWithSecrets_CreatesAGoogleAiStudioEmbeddingsModel() throws IOException {
+        var modelId = "model";
+        var apiKey = "apiKey";
+
+        try (var service = createGoogleAiStudioService()) {
+            var persistedConfig = getPersistedConfigMap(
+                new HashMap<>(Map.of(ServiceFields.MODEL_ID, modelId)),
+                getTaskSettingsMapEmpty(),
+                getSecretSettingsMap(apiKey)
+            );
+
+            var model = service.parsePersistedConfigWithSecrets(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                persistedConfig.config(),
+                persistedConfig.secrets()
+            );
+
+            assertThat(model, instanceOf(GoogleAiStudioEmbeddingsModel.class));
+
+            var embeddingsModel = (GoogleAiStudioEmbeddingsModel) model;
+            assertThat(embeddingsModel.getServiceSettings().modelId(), is(modelId));
+            assertThat(embeddingsModel.getTaskSettings(), is(EmptyTaskSettings.INSTANCE));
+            assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is(apiKey));
+        }
+    }
+
     public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInConfig() throws IOException {
         var modelId = "model";
         var apiKey = "apiKey";
@@ -460,7 +525,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotGoogleAiStudioModel() throws IOEx
         verifyNoMoreInteractions(sender);
     }
 
-    public void testInfer_SendsRequest() throws IOException {
+    public void testInfer_SendsCompletionRequest() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
         try (var service = new GoogleAiStudioService(senderFactory, createWithEmptySettings(threadPool))) {
@@ -508,7 +573,7 @@ public void testInfer_SendsRequest() throws IOException {
 
             webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
 
-            var model = createModel("model", getUrl(webServer), "secret");
+            var model = GoogleAiStudioCompletionModelTests.createModel("model", getUrl(webServer), "secret");
             PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
             service.infer(
                 model,
@@ -541,6 +606,155 @@ public void testInfer_SendsRequest() throws IOException {
         }
     }
 
+    public void testInfer_SendsEmbeddingsRequest() throws IOException {
+        var modelId = "model";
+        var apiKey = "apiKey";
+        var input = "input";
+
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new GoogleAiStudioService(senderFactory, createWithEmptySettings(threadPool))) {
+            String responseJson = """
+                {
+                     "embeddings": [
+                         {
+                             "values": [
+                                 0.0123,
+                                 -0.0123
+                             ]
+                         }
+                     ]
+                 }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = GoogleAiStudioEmbeddingsModelTests.createModel(modelId, apiKey, getUrl(webServer));
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                null,
+                List.of(input),
+                new HashMap<>(),
+                InputType.INGEST,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+            var result = listener.actionGet(TIMEOUT);
+
+            assertThat(result.asMap(), is(buildExpectationFloat(List.of(new float[] { 0.0123F, -0.0123F }))));
+            assertThat(webServer.requests(), hasSize(1));
+            assertThat(webServer.requests().get(0).getUri().getQuery(), endsWith(apiKey));
+            assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), Matchers.equalTo(XContentType.JSON.mediaType()));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            assertThat(requestMap, aMapWithSize(1));
+            assertThat(
+                requestMap.get("requests"),
+                Matchers.is(
+                    List.of(
+                        Map.of(
+                            "model",
+                            Strings.format("%s/%s", "models", modelId),
+                            "content",
+                            Map.of("parts", List.of(Map.of("text", input)))
+                        )
+                    )
+                )
+            );
+        }
+    }
+
+    public void testChunkedInfer_Batches() throws IOException {
+        var modelId = "modelId";
+        var apiKey = "apiKey";
+        var input = List.of("foo", "bar");
+
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new GoogleAiStudioService(senderFactory, createWithEmptySettings(threadPool))) {
+            String responseJson = """
+                {
+                     "embeddings": [
+                         {
+                             "values": [
+                                 0.0123,
+                                 -0.0123
+                             ]
+                         },
+                         {
+                             "values": [
+                                 0.0456,
+                                 -0.0456
+                             ]
+                         }
+                     ]
+                 }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = GoogleAiStudioEmbeddingsModelTests.createModel(modelId, apiKey, getUrl(webServer));
+            PlainActionFuture<List<ChunkedInferenceServiceResults>> listener = new PlainActionFuture<>();
+            service.chunkedInfer(
+                model,
+                input,
+                new HashMap<>(),
+                InputType.INGEST,
+                new ChunkingOptions(null, null),
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var results = listener.actionGet(TIMEOUT);
+            assertThat(results, hasSize(2));
+
+            // first result
+            {
+                assertThat(results.get(0), instanceOf(ChunkedTextEmbeddingFloatResults.class));
+                var floatResult = (ChunkedTextEmbeddingFloatResults) results.get(0);
+                assertThat(floatResult.chunks(), hasSize(1));
+                assertEquals(input.get(0), floatResult.chunks().get(0).matchedText());
+                assertTrue(Arrays.equals(new float[] { 0.0123f, -0.0123f }, floatResult.chunks().get(0).embedding()));
+            }
+
+            // second result
+            {
+                assertThat(results.get(1), instanceOf(ChunkedTextEmbeddingFloatResults.class));
+                var floatResult = (ChunkedTextEmbeddingFloatResults) results.get(1);
+                assertThat(floatResult.chunks(), hasSize(1));
+                assertEquals(input.get(1), floatResult.chunks().get(0).matchedText());
+                assertTrue(Arrays.equals(new float[] { 0.0456f, -0.0456f }, floatResult.chunks().get(0).embedding()));
+            }
+
+            assertThat(webServer.requests(), hasSize(1));
+            assertThat(webServer.requests().get(0).getUri().getQuery(), endsWith(apiKey));
+            assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), Matchers.equalTo(XContentType.JSON.mediaType()));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            assertThat(requestMap, aMapWithSize(1));
+            assertThat(
+                requestMap.get("requests"),
+                is(
+                    List.of(
+                        Map.of(
+                            "model",
+                            Strings.format("%s/%s", "models", modelId),
+                            "content",
+                            Map.of("parts", List.of(Map.of("text", input.get(0))))
+                        ),
+                        Map.of(
+                            "model",
+                            Strings.format("%s/%s", "models", modelId),
+                            "content",
+                            Map.of("parts", List.of(Map.of("text", input.get(1))))
+                        )
+                    )
+                )
+            );
+        }
+    }
+
     public void testInfer_ResourceNotFound() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
@@ -555,7 +769,7 @@ public void testInfer_ResourceNotFound() throws IOException {
                 """;
             webServer.enqueue(new MockResponse().setResponseCode(404).setBody(responseJson));
 
-            var model = createModel("model", getUrl(webServer), "secret");
+            var model = GoogleAiStudioCompletionModelTests.createModel("model", getUrl(webServer), "secret");
             PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
             service.infer(
                 model,
@@ -574,6 +788,132 @@ public void testInfer_ResourceNotFound() throws IOException {
         }
     }
 
+    public void testCheckModelConfig_UpdatesDimensions() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+        var similarityMeasure = SimilarityMeasure.DOT_PRODUCT;
+        var modelId = "model";
+        var apiKey = "apiKey";
+
+        try (var service = new GoogleAiStudioService(senderFactory, createWithEmptySettings(threadPool))) {
+            String responseJson = """
+                {
+                     "embeddings": [
+                         {
+                             "values": [
+                                 0.0123,
+                                 -0.0123
+                             ]
+                         }
+                     ]
+                 }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = GoogleAiStudioEmbeddingsModelTests.createModel(getUrl(webServer), modelId, apiKey, 1, similarityMeasure);
+
+            PlainActionFuture<Model> listener = new PlainActionFuture<>();
+            service.checkModelConfig(model, listener);
+            var result = listener.actionGet(TIMEOUT);
+
+            // Updates dimensions to two as two embeddings were returned instead of one as specified before
+            assertThat(
+                result,
+                is(GoogleAiStudioEmbeddingsModelTests.createModel(getUrl(webServer), modelId, apiKey, 2, similarityMeasure))
+            );
+        }
+    }
+
+    public void testCheckModelConfig_UpdatesSimilarityToDotProduct_WhenItIsNull() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+        var oneDimension = 1;
+        var modelId = "model";
+        var apiKey = "apiKey";
+
+        try (var service = new GoogleAiStudioService(senderFactory, createWithEmptySettings(threadPool))) {
+            String responseJson = """
+                {
+                     "embeddings": [
+                         {
+                             "values": [
+                                 0.0123
+                             ]
+                         }
+                     ]
+                 }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = GoogleAiStudioEmbeddingsModelTests.createModel(getUrl(webServer), modelId, apiKey, oneDimension, null);
+
+            PlainActionFuture<Model> listener = new PlainActionFuture<>();
+            service.checkModelConfig(model, listener);
+            var result = listener.actionGet(TIMEOUT);
+
+            assertThat(
+                result,
+                is(
+                    GoogleAiStudioEmbeddingsModelTests.createModel(
+                        getUrl(webServer),
+                        modelId,
+                        apiKey,
+                        oneDimension,
+                        SimilarityMeasure.DOT_PRODUCT
+                    )
+                )
+            );
+        }
+    }
+
+    public void testCheckModelConfig_DoesNotUpdateSimilarity_WhenItIsSpecifiedAsCosine() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+        var oneDimension = 1;
+        var modelId = "model";
+        var apiKey = "apiKey";
+
+        try (var service = new GoogleAiStudioService(senderFactory, createWithEmptySettings(threadPool))) {
+            String responseJson = """
+                {
+                     "embeddings": [
+                         {
+                             "values": [
+                                 0.0123
+                             ]
+                         }
+                     ]
+                 }
+                """;
+
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = GoogleAiStudioEmbeddingsModelTests.createModel(
+                getUrl(webServer),
+                modelId,
+                apiKey,
+                oneDimension,
+                SimilarityMeasure.COSINE
+            );
+
+            PlainActionFuture<Model> listener = new PlainActionFuture<>();
+            service.checkModelConfig(model, listener);
+            var result = listener.actionGet(TIMEOUT);
+
+            assertThat(
+                result,
+                is(
+                    GoogleAiStudioEmbeddingsModelTests.createModel(
+                        getUrl(webServer),
+                        modelId,
+                        apiKey,
+                        oneDimension,
+                        SimilarityMeasure.COSINE
+                    )
+                )
+            );
+        }
+    }
+
     public static Map<String, Object> buildExpectationCompletions(List<String> completions) {
         return Map.of(
             ChatCompletionResults.COMPLETION,
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModelTests.java
new file mode 100644
index 0000000000000..770b6f8fb4137
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModelTests.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.googleaistudio.embeddings;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.EmptyTaskSettings;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioSecretSettings;
+
+public class GoogleAiStudioEmbeddingsModelTests extends ESTestCase {
+
+    public static GoogleAiStudioEmbeddingsModel createModel(String model, String apiKey, String url) {
+        return new GoogleAiStudioEmbeddingsModel(
+            "id",
+            TaskType.TEXT_EMBEDDING,
+            "service",
+            url,
+            new GoogleAiStudioEmbeddingsServiceSettings(model, null, null, SimilarityMeasure.DOT_PRODUCT, null),
+            EmptyTaskSettings.INSTANCE,
+            new GoogleAiStudioSecretSettings(new SecureString(apiKey.toCharArray()))
+        );
+    }
+
+    public static GoogleAiStudioEmbeddingsModel createModel(
+        String url,
+        String model,
+        String apiKey,
+        Integer dimensions,
+        @Nullable SimilarityMeasure similarityMeasure
+    ) {
+        return new GoogleAiStudioEmbeddingsModel(
+            "id",
+            TaskType.TEXT_EMBEDDING,
+            "service",
+            url,
+            new GoogleAiStudioEmbeddingsServiceSettings(model, null, dimensions, similarityMeasure, null),
+            EmptyTaskSettings.INSTANCE,
+            new GoogleAiStudioSecretSettings(new SecureString(apiKey.toCharArray()))
+        );
+    }
+
+    public static GoogleAiStudioEmbeddingsModel createModel(
+        String model,
+        String apiKey,
+        @Nullable Integer tokenLimit,
+        @Nullable Integer dimensions
+    ) {
+        return new GoogleAiStudioEmbeddingsModel(
+            "id",
+            TaskType.TEXT_EMBEDDING,
+            "service",
+            new GoogleAiStudioEmbeddingsServiceSettings(model, tokenLimit, dimensions, SimilarityMeasure.DOT_PRODUCT, null),
+            EmptyTaskSettings.INSTANCE,
+            new GoogleAiStudioSecretSettings(new SecureString(apiKey.toCharArray()))
+        );
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettingsTests.java
new file mode 100644
index 0000000000000..b5fbd28b476ba
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettingsTests.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.googleaistudio.embeddings;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ServiceFields;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString;
+import static org.elasticsearch.xpack.inference.Utils.randomSimilarityMeasure;
+import static org.hamcrest.Matchers.is;
+
+public class GoogleAiStudioEmbeddingsServiceSettingsTests extends AbstractWireSerializingTestCase<GoogleAiStudioEmbeddingsServiceSettings> {
+
+    private static GoogleAiStudioEmbeddingsServiceSettings createRandom() {
+        return new GoogleAiStudioEmbeddingsServiceSettings(
+            randomAlphaOfLength(8),
+            randomFrom(randomNonNegativeInt(), null),
+            randomFrom(randomNonNegativeInt(), null),
+            randomFrom(randomSimilarityMeasure(), null),
+            randomFrom(RateLimitSettingsTests.createRandom(), null)
+        );
+    }
+
+    public void testFromMap_Request_CreatesSettingsCorrectly() {
+        var model = randomAlphaOfLength(8);
+        var maxInputTokens = randomIntBetween(1, 1024);
+        var dims = randomIntBetween(1, 10000);
+        var similarity = randomSimilarityMeasure();
+
+        var serviceSettings = GoogleAiStudioEmbeddingsServiceSettings.fromMap(
+            new HashMap<>(
+                Map.of(
+                    ServiceFields.MODEL_ID,
+                    model,
+                    ServiceFields.MAX_INPUT_TOKENS,
+                    maxInputTokens,
+                    ServiceFields.DIMENSIONS,
+                    dims,
+                    ServiceFields.SIMILARITY,
+                    similarity.toString()
+                )
+            )
+        );
+
+        assertThat(serviceSettings, is(new GoogleAiStudioEmbeddingsServiceSettings(model, maxInputTokens, dims, similarity, null)));
+    }
+
+    public void testToXContent_WritesAllValues() throws IOException {
+        var entity = new GoogleAiStudioEmbeddingsServiceSettings("model", 1024, 8, SimilarityMeasure.DOT_PRODUCT, null);
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "model_id":"model",
+                "max_input_tokens": 1024,
+                "dimensions": 8,
+                "similarity": "dot_product",
+                "rate_limit": {
+                    "requests_per_minute":360
+                }
+            }"""));
+    }
+
+    public void testToFilteredXContent_WritesAllValues_Except_RateLimit() throws IOException {
+        var entity = new GoogleAiStudioEmbeddingsServiceSettings("model", 1024, 8, SimilarityMeasure.DOT_PRODUCT, null);
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        var filteredXContent = entity.getFilteredXContentObject();
+        filteredXContent.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
+            {
+                "model_id":"model",
+                "max_input_tokens": 1024,
+                "dimensions": 8,
+                "similarity": "dot_product"
+            }"""));
+    }
+
+    @Override
+    protected Writeable.Reader<GoogleAiStudioEmbeddingsServiceSettings> instanceReader() {
+        return GoogleAiStudioEmbeddingsServiceSettings::new;
+    }
+
+    @Override
+    protected GoogleAiStudioEmbeddingsServiceSettings createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected GoogleAiStudioEmbeddingsServiceSettings mutateInstance(GoogleAiStudioEmbeddingsServiceSettings instance) throws IOException {
+        return randomValueOtherThan(instance, GoogleAiStudioEmbeddingsServiceSettingsTests::createRandom);
+    }
+}

From 1506aa18acea0fff63dbba04bdc8f9832fc1d2fe Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Mon, 3 Jun 2024 09:50:15 -0400
Subject: [PATCH 109/208] [ESQL] Migrate QueryPlanTests out of core (#109291)

Query plan tests were using the core version of Add. This move enables them to use the ESQL version, and thus actually exercise the code we're running.
---
 .../org/elasticsearch/xpack/esql}/plan/QueryPlanTests.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
 rename x-pack/plugin/{esql-core/src/test/java/org/elasticsearch/xpack/esql/core => esql/src/test/java/org/elasticsearch/xpack/esql}/plan/QueryPlanTests.java (96%)

diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/plan/QueryPlanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java
similarity index 96%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/plan/QueryPlanTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java
index 747823795d408..11c2d9532ff16 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/plan/QueryPlanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.plan;
+package org.elasticsearch.xpack.esql.plan;
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
@@ -13,12 +13,12 @@
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.core.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
-import org.elasticsearch.xpack.esql.core.plan.logical.Project;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
+import org.elasticsearch.xpack.esql.plan.logical.Project;
 
 import java.util.ArrayList;
 import java.util.List;

From 62c39b7e05978bbcf75a9c940d5dfeda2aa01ad8 Mon Sep 17 00:00:00 2001
From: "Mark J. Hoy" <mark.hoy@elastic.co>
Date: Mon, 3 Jun 2024 10:54:16 -0400
Subject: [PATCH 110/208] [Inference API] Add Mistral Embeddings Support to
 Inference API (#109194)

* initial commit; need to write tests

* add service, model, and service settings tests

* fix compilation errors

* complete tests

* use batches for chunked input; cleanups

* update transport versions
---
 .../org/elasticsearch/TransportVersions.java  |   1 +
 .../InferenceNamedWriteablesProvider.java     |  14 +
 .../xpack/inference/InferencePlugin.java      |   2 +
 .../action/mistral/MistralAction.java         |  45 ++
 .../action/mistral/MistralActionCreator.java  |  40 ++
 .../action/mistral/MistralActionVisitor.java  |  17 +
 ...eAiStudioChatCompletionRequestManager.java |   8 +-
 ...AzureAiStudioEmbeddingsRequestManager.java |   8 +-
 .../MistralEmbeddingsRequestManager.java      |  75 ++
 .../mistral/MistralEmbeddingsRequest.java     |  76 ++
 .../MistralEmbeddingsRequestEntity.java       |  39 ++
 ...zureMistralOpenAiErrorResponseEntity.java} |   6 +-
 ...MistralOpenAiExternalResponseHandler.java} |   6 +-
 .../MistralEmbeddingsResponseEntity.java      |  24 +
 .../services/mistral/MistralConstants.java    |  21 +
 .../services/mistral/MistralService.java      | 272 ++++++++
 .../embeddings/MistralEmbeddingsModel.java    | 134 ++++
 .../MistralEmbeddingsServiceSettings.java     | 182 +++++
 .../MistralEmbeddingsRequestEntityTests.java  |  32 +
 .../MistralEmbeddingsRequestTests.java        |  81 +++
 ...zureAndOpenAiErrorResponseEntityTests.java |   6 +-
 ...AndOpenAiExternalResponseHandlerTests.java |  54 +-
 .../services/mistral/MistralServiceTests.java | 651 ++++++++++++++++++
 .../MistralEmbeddingModelTests.java           |  42 ++
 ...MistralEmbeddingsServiceSettingsTests.java | 149 ++++
 25 files changed, 1941 insertions(+), 44 deletions(-)
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralAction.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionVisitor.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/MistralEmbeddingsRequestManager.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequest.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestEntity.java
 rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/{AzureAndOpenAiErrorResponseEntity.java => AzureMistralOpenAiErrorResponseEntity.java} (90%)
 rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/{AzureAndOpenAiExternalResponseHandler.java => AzureMistralOpenAiExternalResponseHandler.java} (96%)
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/mistral/MistralEmbeddingsResponseEntity.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralConstants.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java
 create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestEntityTests.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestTests.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingModelTests.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettingsTests.java

diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index b0d14b99f15cd..07579161a85c8 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -182,6 +182,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion WATCHER_REQUEST_TIMEOUTS = def(8_673_00_0);
     public static final TransportVersion ML_INFERENCE_ENHANCE_DELETE_ENDPOINT = def(8_674_00_0);
     public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_EMBEDDINGS_ADDED = def(8_675_00_0);
+    public static final TransportVersion ADD_MISTRAL_EMBEDDINGS_INFERENCE = def(8_676_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
index 9bdc6f148917a..506cef4905875 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceServiceSettings;
 import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserSecretSettings;
 import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserServiceSettings;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettings;
 import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionServiceSettings;
 import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionTaskSettings;
 import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsServiceSettings;
@@ -109,10 +110,23 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         addAzureOpenAiNamedWriteables(namedWriteables);
         addAzureAiStudioNamedWriteables(namedWriteables);
         addGoogleAiStudioNamedWritables(namedWriteables);
+        addMistralNamedWriteables(namedWriteables);
 
         return namedWriteables;
     }
 
+    private static void addMistralNamedWriteables(List<NamedWriteableRegistry.Entry> namedWriteables) {
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(
+                ServiceSettings.class,
+                MistralEmbeddingsServiceSettings.NAME,
+                MistralEmbeddingsServiceSettings::new
+            )
+        );
+
+        // note - no task settings for Mistral embeddings...
+    }
+
     private static void addAzureAiStudioNamedWriteables(List<NamedWriteableRegistry.Entry> namedWriteables) {
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
index 9a9b11fd1400e..1e0f715e3f3e9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java
@@ -74,6 +74,7 @@
 import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioService;
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceService;
 import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserService;
+import org.elasticsearch.xpack.inference.services.mistral.MistralService;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiService;
 
 import java.util.ArrayList;
@@ -198,6 +199,7 @@ public List<InferenceServiceExtension.Factory> getInferenceServiceFactories() {
             context -> new AzureOpenAiService(httpFactory.get(), serviceComponents.get()),
             context -> new AzureAiStudioService(httpFactory.get(), serviceComponents.get()),
             context -> new GoogleAiStudioService(httpFactory.get(), serviceComponents.get()),
+            context -> new MistralService(httpFactory.get(), serviceComponents.get()),
             ElasticsearchInternalService::new
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralAction.java
new file mode 100644
index 0000000000000..f7b51e80a04b3
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralAction.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.mistral;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs;
+import org.elasticsearch.xpack.inference.external.http.sender.MistralEmbeddingsRequestManager;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.createInternalServerError;
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.wrapFailuresInElasticsearchException;
+
+public class MistralAction implements ExecutableAction {
+    protected final Sender sender;
+    protected final MistralEmbeddingsRequestManager requestCreator;
+    protected final String errorMessage;
+
+    protected MistralAction(Sender sender, MistralEmbeddingsRequestManager requestCreator, String errorMessage) {
+        this.sender = sender;
+        this.requestCreator = requestCreator;
+        this.errorMessage = errorMessage;
+    }
+
+    @Override
+    public void execute(InferenceInputs inferenceInputs, TimeValue timeout, ActionListener<InferenceServiceResults> listener) {
+        try {
+            ActionListener<InferenceServiceResults> wrappedListener = wrapFailuresInElasticsearchException(errorMessage, listener);
+
+            sender.send(requestCreator, inferenceInputs, timeout, wrappedListener);
+        } catch (ElasticsearchException e) {
+            listener.onFailure(e);
+        } catch (Exception e) {
+            listener.onFailure(createInternalServerError(e, errorMessage));
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java
new file mode 100644
index 0000000000000..a023973ea6aa5
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionCreator.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.mistral;
+
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.http.sender.MistralEmbeddingsRequestManager;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsModel;
+
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage;
+
+public class MistralActionCreator implements MistralActionVisitor {
+    private final Sender sender;
+    private final ServiceComponents serviceComponents;
+
+    public MistralActionCreator(Sender sender, ServiceComponents serviceComponents) {
+        this.sender = Objects.requireNonNull(sender);
+        this.serviceComponents = Objects.requireNonNull(serviceComponents);
+    }
+
+    @Override
+    public ExecutableAction create(MistralEmbeddingsModel embeddingsModel, Map<String, Object> taskSettings) {
+        var requestManager = new MistralEmbeddingsRequestManager(
+            embeddingsModel,
+            serviceComponents.truncator(),
+            serviceComponents.threadPool()
+        );
+        var errorMessage = constructFailedToSendRequestMessage(embeddingsModel.uri(), "Mistral embeddings");
+        return new MistralAction(sender, requestManager, errorMessage);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionVisitor.java
new file mode 100644
index 0000000000000..3764efeb0f6c8
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/mistral/MistralActionVisitor.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.action.mistral;
+
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsModel;
+
+import java.util.Map;
+
+public interface MistralActionVisitor {
+    ExecutableAction create(MistralEmbeddingsModel embeddingsModel, Map<String, Object> taskSettings);
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java
index 76ef37592d88e..deff410aebaa8 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
 import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
 import org.elasticsearch.xpack.inference.external.request.azureaistudio.AzureAiStudioChatCompletionRequest;
-import org.elasticsearch.xpack.inference.external.response.AzureAndOpenAiErrorResponseEntity;
-import org.elasticsearch.xpack.inference.external.response.AzureAndOpenAiExternalResponseHandler;
+import org.elasticsearch.xpack.inference.external.response.AzureMistralOpenAiErrorResponseEntity;
+import org.elasticsearch.xpack.inference.external.response.AzureMistralOpenAiExternalResponseHandler;
 import org.elasticsearch.xpack.inference.external.response.azureaistudio.AzureAiStudioChatCompletionResponseEntity;
 import org.elasticsearch.xpack.inference.services.azureaistudio.completion.AzureAiStudioChatCompletionModel;
 
@@ -51,10 +51,10 @@ public Runnable create(
     }
 
     private static ResponseHandler createCompletionHandler() {
-        return new AzureAndOpenAiExternalResponseHandler(
+        return new AzureMistralOpenAiExternalResponseHandler(
             "azure ai studio completion",
             new AzureAiStudioChatCompletionResponseEntity(),
-            AzureAndOpenAiErrorResponseEntity::fromResponse
+            AzureMistralOpenAiErrorResponseEntity::fromResponse
         );
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioEmbeddingsRequestManager.java
index c2edc79dfe937..a2b363151a417 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioEmbeddingsRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioEmbeddingsRequestManager.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
 import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
 import org.elasticsearch.xpack.inference.external.request.azureaistudio.AzureAiStudioEmbeddingsRequest;
-import org.elasticsearch.xpack.inference.external.response.AzureAndOpenAiErrorResponseEntity;
-import org.elasticsearch.xpack.inference.external.response.AzureAndOpenAiExternalResponseHandler;
+import org.elasticsearch.xpack.inference.external.response.AzureMistralOpenAiErrorResponseEntity;
+import org.elasticsearch.xpack.inference.external.response.AzureMistralOpenAiExternalResponseHandler;
 import org.elasticsearch.xpack.inference.external.response.azureaistudio.AzureAiStudioEmbeddingsResponseEntity;
 import org.elasticsearch.xpack.inference.services.azureaistudio.embeddings.AzureAiStudioEmbeddingsModel;
 
@@ -55,10 +55,10 @@ public Runnable create(
     }
 
     private static ResponseHandler createEmbeddingsHandler() {
-        return new AzureAndOpenAiExternalResponseHandler(
+        return new AzureMistralOpenAiExternalResponseHandler(
             "azure ai studio text embedding",
             new AzureAiStudioEmbeddingsResponseEntity(),
-            AzureAndOpenAiErrorResponseEntity::fromResponse
+            AzureMistralOpenAiErrorResponseEntity::fromResponse
         );
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/MistralEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/MistralEmbeddingsRequestManager.java
new file mode 100644
index 0000000000000..f31a633581705
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/MistralEmbeddingsRequestManager.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.common.Truncator;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
+import org.elasticsearch.xpack.inference.external.request.mistral.MistralEmbeddingsRequest;
+import org.elasticsearch.xpack.inference.external.response.AzureMistralOpenAiErrorResponseEntity;
+import org.elasticsearch.xpack.inference.external.response.AzureMistralOpenAiExternalResponseHandler;
+import org.elasticsearch.xpack.inference.external.response.mistral.MistralEmbeddingsResponseEntity;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsModel;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xpack.inference.common.Truncator.truncate;
+
+public class MistralEmbeddingsRequestManager extends BaseRequestManager {
+    private static final Logger logger = LogManager.getLogger(AzureOpenAiEmbeddingsRequestManager.class);
+    private static final ResponseHandler HANDLER = createEmbeddingsHandler();
+
+    private final Truncator truncator;
+    private final MistralEmbeddingsModel model;
+
+    private static ResponseHandler createEmbeddingsHandler() {
+        return new AzureMistralOpenAiExternalResponseHandler(
+            "mistral text embedding",
+            new MistralEmbeddingsResponseEntity(),
+            AzureMistralOpenAiErrorResponseEntity::fromResponse
+        );
+    }
+
+    public MistralEmbeddingsRequestManager(MistralEmbeddingsModel model, Truncator truncator, ThreadPool threadPool) {
+        super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model), model.rateLimitSettings());
+        this.model = Objects.requireNonNull(model);
+        this.truncator = Objects.requireNonNull(truncator);
+
+    }
+
+    @Override
+    public Runnable create(
+        String query,
+        List<String> input,
+        RequestSender requestSender,
+        Supplier<Boolean> hasRequestCompletedFunction,
+        HttpClientContext context,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        var truncatedInput = truncate(input, model.getServiceSettings().maxInputTokens());
+        MistralEmbeddingsRequest request = new MistralEmbeddingsRequest(truncator, truncatedInput, model);
+
+        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+    }
+
+    record RateLimitGrouping(int keyHashCode) {
+        public static RateLimitGrouping of(MistralEmbeddingsModel model) {
+            Objects.requireNonNull(model);
+
+            return new RateLimitGrouping(model.getSecretSettings().apiKey().hashCode());
+        }
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequest.java
new file mode 100644
index 0000000000000..e1c90c08f643f
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequest.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.mistral;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.common.Truncator;
+import org.elasticsearch.xpack.inference.external.request.HttpRequest;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsModel;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+
+import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader;
+
+public class MistralEmbeddingsRequest implements Request {
+    private final URI uri;
+    private final MistralEmbeddingsModel embeddingsModel;
+    private final String inferenceEntityId;
+    private final Truncator.TruncationResult truncationResult;
+    private final Truncator truncator;
+
+    public MistralEmbeddingsRequest(Truncator truncator, Truncator.TruncationResult input, MistralEmbeddingsModel model) {
+        this.uri = model.uri();
+        this.embeddingsModel = model;
+        this.inferenceEntityId = model.getInferenceEntityId();
+        this.truncator = truncator;
+        this.truncationResult = input;
+    }
+
+    @Override
+    public HttpRequest createHttpRequest() {
+        HttpPost httpPost = new HttpPost(this.uri);
+
+        ByteArrayEntity byteEntity = new ByteArrayEntity(
+            Strings.toString(new MistralEmbeddingsRequestEntity(embeddingsModel.model(), truncationResult.input()))
+                .getBytes(StandardCharsets.UTF_8)
+        );
+        httpPost.setEntity(byteEntity);
+
+        httpPost.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType());
+        httpPost.setHeader(createAuthBearerHeader(embeddingsModel.getSecretSettings().apiKey()));
+
+        return new HttpRequest(httpPost, getInferenceEntityId());
+    }
+
+    @Override
+    public URI getURI() {
+        return uri;
+    }
+
+    @Override
+    public Request truncate() {
+        var truncatedInput = truncator.truncate(truncationResult.input());
+        return new MistralEmbeddingsRequest(truncator, truncatedInput, embeddingsModel);
+    }
+
+    @Override
+    public boolean[] getTruncationInfo() {
+        return truncationResult.truncated().clone();
+    }
+
+    @Override
+    public String getInferenceEntityId() {
+        return inferenceEntityId;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestEntity.java
new file mode 100644
index 0000000000000..d852e9ee34046
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestEntity.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.mistral;
+
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.mistral.MistralConstants.ENCODING_FORMAT_FIELD;
+import static org.elasticsearch.xpack.inference.services.mistral.MistralConstants.INPUT_FIELD;
+import static org.elasticsearch.xpack.inference.services.mistral.MistralConstants.MODEL_FIELD;
+
+public record MistralEmbeddingsRequestEntity(String model, List<String> input) implements ToXContentObject {
+    public MistralEmbeddingsRequestEntity {
+        Objects.requireNonNull(model);
+        Objects.requireNonNull(input);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+
+        builder.field(MODEL_FIELD, model);
+        builder.field(INPUT_FIELD, input);
+        builder.field(ENCODING_FORMAT_FIELD, "float");
+
+        builder.endObject();
+
+        return builder;
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiErrorResponseEntity.java
similarity index 90%
rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiErrorResponseEntity.java
rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiErrorResponseEntity.java
index 4ac77d6df3c33..83ea7801dfd58 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiErrorResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiErrorResponseEntity.java
@@ -31,10 +31,10 @@
  * This currently covers error handling for Azure AI Studio, however this pattern
  * can be used to simplify and refactor handling for Azure OpenAI and OpenAI responses.
  */
-public class AzureAndOpenAiErrorResponseEntity implements ErrorMessage {
+public class AzureMistralOpenAiErrorResponseEntity implements ErrorMessage {
     protected String errorMessage;
 
-    public AzureAndOpenAiErrorResponseEntity(String errorMessage) {
+    public AzureMistralOpenAiErrorResponseEntity(String errorMessage) {
         this.errorMessage = errorMessage;
     }
 
@@ -62,7 +62,7 @@ public static ErrorMessage fromResponse(HttpResult response) {
             if (error != null) {
                 var message = (String) error.get("message");
                 if (message != null) {
-                    return new AzureAndOpenAiErrorResponseEntity(message);
+                    return new AzureMistralOpenAiErrorResponseEntity(message);
                 }
             }
         } catch (Exception e) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiExternalResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java
similarity index 96%
rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiExternalResponseHandler.java
rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java
index 5f803ad6fe74e..dfdb6712d5e45 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiExternalResponseHandler.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java
@@ -29,7 +29,7 @@
  * This currently covers response handling for Azure AI Studio, however this pattern
  * can be used to simplify and refactor handling for Azure OpenAI and OpenAI responses.
  */
-public class AzureAndOpenAiExternalResponseHandler extends BaseResponseHandler {
+public class AzureMistralOpenAiExternalResponseHandler extends BaseResponseHandler {
 
     // The maximum number of requests that are permitted before exhausting the rate limit.
     static final String REQUESTS_LIMIT = "x-ratelimit-limit-requests";
@@ -43,7 +43,7 @@ public class AzureAndOpenAiExternalResponseHandler extends BaseResponseHandler {
     static final String CONTENT_TOO_LARGE_MESSAGE = "Please reduce your prompt; or completion length.";
     static final String SERVER_BUSY_ERROR = "Received a server busy error status code";
 
-    public AzureAndOpenAiExternalResponseHandler(
+    public AzureMistralOpenAiExternalResponseHandler(
         String requestType,
         ResponseParser parseFunction,
         Function<HttpResult, ErrorMessage> errorParseFunction
@@ -116,7 +116,7 @@ public static boolean isContentTooLarge(HttpResult result) {
         }
 
         if (statusCode == 400) {
-            var errorEntity = AzureAndOpenAiErrorResponseEntity.fromResponse(result);
+            var errorEntity = AzureMistralOpenAiErrorResponseEntity.fromResponse(result);
             return errorEntity != null && errorEntity.getErrorMessage().contains(CONTENT_TOO_LARGE_MESSAGE);
         }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/mistral/MistralEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/mistral/MistralEmbeddingsResponseEntity.java
new file mode 100644
index 0000000000000..01de92c20a3f1
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/mistral/MistralEmbeddingsResponseEntity.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.response.mistral;
+
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.xpack.inference.external.http.HttpResult;
+import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.response.BaseResponseEntity;
+import org.elasticsearch.xpack.inference.external.response.openai.OpenAiEmbeddingsResponseEntity;
+
+import java.io.IOException;
+
+public class MistralEmbeddingsResponseEntity extends BaseResponseEntity {
+    @Override
+    protected InferenceServiceResults fromResponse(Request request, HttpResult response) throws IOException {
+        // expected response type is the same as the Open AI Embeddings
+        return OpenAiEmbeddingsResponseEntity.fromResponse(request, response);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralConstants.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralConstants.java
new file mode 100644
index 0000000000000..d059545ca1ea3
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralConstants.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.mistral;
+
+public class MistralConstants {
+    public static final String API_EMBEDDINGS_PATH = "https://api.mistral.ai/v1/embeddings";
+
+    // note - there is no bounds information available from Mistral,
+    // so we'll use a sane default here which is the same as Cohere's
+    public static final int MAX_BATCH_SIZE = 96;
+
+    public static final String API_KEY_FIELD = "api_key";
+    public static final String MODEL_FIELD = "model";
+    public static final String INPUT_FIELD = "input";
+    public static final String ENCODING_FORMAT_FIELD = "encoding_format";
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java
new file mode 100644
index 0000000000000..7ddb71d001e8c
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java
@@ -0,0 +1,272 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.mistral;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.ChunkedInferenceServiceResults;
+import org.elasticsearch.inference.ChunkingOptions;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingResults;
+import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
+import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
+import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
+import org.elasticsearch.xpack.inference.common.EmbeddingRequestChunker;
+import org.elasticsearch.xpack.inference.external.action.mistral.MistralActionCreator;
+import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.SenderService;
+import org.elasticsearch.xpack.inference.services.ServiceComponents;
+import org.elasticsearch.xpack.inference.services.ServiceUtils;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettings;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.TransportVersions.ADD_MISTRAL_EMBEDDINGS_INFERENCE;
+import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
+
+public class MistralService extends SenderService {
+    public static final String NAME = "mistral";
+
+    public MistralService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
+        super(factory, serviceComponents);
+    }
+
+    @Override
+    protected void doInfer(
+        Model model,
+        List<String> input,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        TimeValue timeout,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        var actionCreator = new MistralActionCreator(getSender(), getServiceComponents());
+
+        if (model instanceof MistralEmbeddingsModel mistralEmbeddingsModel) {
+            var action = mistralEmbeddingsModel.accept(actionCreator, taskSettings);
+            action.execute(new DocumentsOnlyInput(input), timeout, listener);
+        } else {
+            listener.onFailure(createInvalidModelException(model));
+        }
+    }
+
+    @Override
+    protected void doInfer(
+        Model model,
+        String query,
+        List<String> input,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        TimeValue timeout,
+        ActionListener<InferenceServiceResults> listener
+    ) {
+        throw new UnsupportedOperationException("Mistral service does not support inference with query input");
+    }
+
+    @Override
+    protected void doChunkedInfer(
+        Model model,
+        String query,
+        List<String> input,
+        Map<String, Object> taskSettings,
+        InputType inputType,
+        ChunkingOptions chunkingOptions,
+        TimeValue timeout,
+        ActionListener<List<ChunkedInferenceServiceResults>> listener
+    ) {
+        var actionCreator = new MistralActionCreator(getSender(), getServiceComponents());
+
+        if (model instanceof MistralEmbeddingsModel mistralEmbeddingsModel) {
+            var batchedRequests = new EmbeddingRequestChunker(input, MistralConstants.MAX_BATCH_SIZE).batchRequestsWithListeners(listener);
+
+            for (var request : batchedRequests) {
+                var action = mistralEmbeddingsModel.accept(actionCreator, taskSettings);
+                action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener());
+            }
+        } else {
+            listener.onFailure(createInvalidModelException(model));
+        }
+    }
+
+    private static List<ChunkedInferenceServiceResults> translateToChunkedResults(
+        List<String> inputs,
+        InferenceServiceResults inferenceResults
+    ) {
+        if (inferenceResults instanceof TextEmbeddingResults textEmbeddingResults) {
+            return ChunkedTextEmbeddingResults.of(inputs, textEmbeddingResults);
+        } else if (inferenceResults instanceof ErrorInferenceResults error) {
+            return List.of(new ErrorChunkedInferenceResults(error.getException()));
+        } else {
+            throw createInvalidChunkedResultException(inferenceResults.getWriteableName());
+        }
+    }
+
+    @Override
+    public String name() {
+        return NAME;
+    }
+
+    @Override
+    public void parseRequestConfig(
+        String modelId,
+        TaskType taskType,
+        Map<String, Object> config,
+        Set<String> platfromArchitectures,
+        ActionListener<Model> parsedModelListener
+    ) {
+        try {
+            Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+            Map<String, Object> taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS);
+
+            MistralEmbeddingsModel model = createModel(
+                modelId,
+                taskType,
+                serviceSettingsMap,
+                taskSettingsMap,
+                serviceSettingsMap,
+                TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME),
+                ConfigurationParseContext.REQUEST
+            );
+
+            throwIfNotEmptyMap(config, NAME);
+            throwIfNotEmptyMap(serviceSettingsMap, NAME);
+            throwIfNotEmptyMap(taskSettingsMap, NAME);
+
+            parsedModelListener.onResponse(model);
+        } catch (Exception e) {
+            parsedModelListener.onFailure(e);
+        }
+    }
+
+    @Override
+    public Model parsePersistedConfigWithSecrets(
+        String modelId,
+        TaskType taskType,
+        Map<String, Object> config,
+        Map<String, Object> secrets
+    ) {
+        Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+        Map<String, Object> taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS);
+        Map<String, Object> secretSettingsMap = removeFromMapOrDefaultEmpty(secrets, ModelSecrets.SECRET_SETTINGS);
+
+        return createModelFromPersistent(
+            modelId,
+            taskType,
+            serviceSettingsMap,
+            taskSettingsMap,
+            secretSettingsMap,
+            parsePersistedConfigErrorMsg(modelId, NAME)
+        );
+    }
+
+    @Override
+    public Model parsePersistedConfig(String modelId, TaskType taskType, Map<String, Object> config) {
+        Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
+        Map<String, Object> taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS);
+
+        return createModelFromPersistent(
+            modelId,
+            taskType,
+            serviceSettingsMap,
+            taskSettingsMap,
+            null,
+            parsePersistedConfigErrorMsg(modelId, NAME)
+        );
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return ADD_MISTRAL_EMBEDDINGS_INFERENCE;
+    }
+
+    private static MistralEmbeddingsModel createModel(
+        String modelId,
+        TaskType taskType,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secretSettings,
+        String failureMessage,
+        ConfigurationParseContext context
+    ) {
+        if (taskType == TaskType.TEXT_EMBEDDING) {
+            return new MistralEmbeddingsModel(modelId, taskType, NAME, serviceSettings, taskSettings, secretSettings, context);
+        }
+
+        throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
+    }
+
+    private MistralEmbeddingsModel createModelFromPersistent(
+        String inferenceEntityId,
+        TaskType taskType,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        Map<String, Object> secretSettings,
+        String failureMessage
+    ) {
+        return createModel(
+            inferenceEntityId,
+            taskType,
+            serviceSettings,
+            taskSettings,
+            secretSettings,
+            failureMessage,
+            ConfigurationParseContext.PERSISTENT
+        );
+    }
+
+    @Override
+    public void checkModelConfig(Model model, ActionListener<Model> listener) {
+        if (model instanceof MistralEmbeddingsModel embeddingsModel) {
+            ServiceUtils.getEmbeddingSize(
+                model,
+                this,
+                listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateEmbeddingModelConfig(embeddingsModel, size)))
+            );
+        } else {
+            listener.onResponse(model);
+        }
+    }
+
+    private MistralEmbeddingsModel updateEmbeddingModelConfig(MistralEmbeddingsModel embeddingsModel, int embeddingsSize) {
+        var embeddingServiceSettings = embeddingsModel.getServiceSettings();
+
+        var similarityFromModel = embeddingsModel.getServiceSettings().similarity();
+        var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel;
+
+        MistralEmbeddingsServiceSettings serviceSettings = new MistralEmbeddingsServiceSettings(
+            embeddingServiceSettings.model(),
+            embeddingsSize,
+            embeddingServiceSettings.maxInputTokens(),
+            similarityToUse,
+            embeddingServiceSettings.rateLimitSettings()
+        );
+
+        return new MistralEmbeddingsModel(embeddingsModel, serviceSettings);
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java
new file mode 100644
index 0000000000000..c3d261efea79a
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.mistral.embeddings;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.EmptyTaskSettings;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.TaskSettings;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
+import org.elasticsearch.xpack.inference.external.action.mistral.MistralActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.inference.services.mistral.MistralConstants.API_EMBEDDINGS_PATH;
+
+public class MistralEmbeddingsModel extends Model {
+    protected String model;
+    protected URI uri;
+    protected RateLimitSettings rateLimitSettings;
+
+    public MistralEmbeddingsModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        String service,
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
+    ) {
+        this(
+            inferenceEntityId,
+            taskType,
+            service,
+            MistralEmbeddingsServiceSettings.fromMap(serviceSettings, context),
+            EmptyTaskSettings.INSTANCE,    // no task settings for Mistral embeddings
+            DefaultSecretSettings.fromMap(secrets)
+        );
+    }
+
+    public MistralEmbeddingsModel(MistralEmbeddingsModel model, TaskSettings taskSettings, RateLimitSettings rateLimitSettings) {
+        super(model, taskSettings);
+        this.model = Objects.requireNonNull(model.model);
+        this.rateLimitSettings = Objects.requireNonNull(rateLimitSettings);
+        setEndpointUrl();
+    }
+
+    public MistralEmbeddingsModel(MistralEmbeddingsModel model, MistralEmbeddingsServiceSettings serviceSettings) {
+        super(model, serviceSettings);
+        setPropertiesFromServiceSettings(serviceSettings);
+    }
+
+    protected MistralEmbeddingsModel(ModelConfigurations modelConfigurations, ModelSecrets modelSecrets) {
+        super(modelConfigurations, modelSecrets);
+        setPropertiesFromServiceSettings((MistralEmbeddingsServiceSettings) modelConfigurations.getServiceSettings());
+    }
+
+    public MistralEmbeddingsModel(
+        String inferenceEntityId,
+        TaskType taskType,
+        String service,
+        MistralEmbeddingsServiceSettings serviceSettings,
+        TaskSettings taskSettings,
+        DefaultSecretSettings secrets
+    ) {
+        super(
+            new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, new EmptyTaskSettings()),
+            new ModelSecrets(secrets)
+        );
+        setPropertiesFromServiceSettings(serviceSettings);
+    }
+
+    private void setPropertiesFromServiceSettings(MistralEmbeddingsServiceSettings serviceSettings) {
+        this.model = serviceSettings.model();
+        this.rateLimitSettings = serviceSettings.rateLimitSettings();
+        setEndpointUrl();
+    }
+
+    @Override
+    public MistralEmbeddingsServiceSettings getServiceSettings() {
+        return (MistralEmbeddingsServiceSettings) super.getServiceSettings();
+    }
+
+    public String model() {
+        return this.model;
+    }
+
+    public URI uri() {
+        return this.uri;
+    }
+
+    public RateLimitSettings rateLimitSettings() {
+        return this.rateLimitSettings;
+    }
+
+    private void setEndpointUrl() {
+        try {
+            this.uri = new URI(API_EMBEDDINGS_PATH);
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    // Needed for testing only
+    public void setURI(String newUri) {
+        try {
+            this.uri = new URI(newUri);
+        } catch (URISyntaxException e) {
+            // swallow any error
+        }
+    }
+
+    @Override
+    public DefaultSecretSettings getSecretSettings() {
+        return (DefaultSecretSettings) super.getSecretSettings();
+    }
+
+    public ExecutableAction accept(MistralActionVisitor creator, Map<String, Object> taskSettings) {
+        return creator.create(this, taskSettings);
+    }
+}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java
new file mode 100644
index 0000000000000..d2ea8ccbd18bd
--- /dev/null
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java
@@ -0,0 +1,182 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.mistral.embeddings;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ServiceSettings;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.TransportVersions.ADD_MISTRAL_EMBEDDINGS_INFERENCE;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS;
+import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeAsType;
+import static org.elasticsearch.xpack.inference.services.mistral.MistralConstants.MODEL_FIELD;
+
+public class MistralEmbeddingsServiceSettings extends FilteredXContentObject implements ServiceSettings {
+    public static final String NAME = "mistral_embeddings_service_settings";
+
+    private final String model;
+    private final Integer dimensions;
+    private final SimilarityMeasure similarity;
+    private final Integer maxInputTokens;
+    private final RateLimitSettings rateLimitSettings;
+
+    // default for Mistral is 5 requests / sec
+    // setting this to 240 (4 requests / sec) is a sane default for us
+    protected static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(240);
+
+    public static MistralEmbeddingsServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
+        ValidationException validationException = new ValidationException();
+
+        String model = extractRequiredString(map, MODEL_FIELD, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        SimilarityMeasure similarity = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException);
+        Integer maxInputTokens = extractOptionalPositiveInteger(
+            map,
+            MAX_INPUT_TOKENS,
+            ModelConfigurations.SERVICE_SETTINGS,
+            validationException
+        );
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        Integer dims = removeAsType(map, DIMENSIONS, Integer.class);
+
+        if (validationException.validationErrors().isEmpty() == false) {
+            throw validationException;
+        }
+
+        return new MistralEmbeddingsServiceSettings(model, dims, maxInputTokens, similarity, rateLimitSettings);
+    }
+
+    public MistralEmbeddingsServiceSettings(StreamInput in) throws IOException {
+        this.model = in.readString();
+        this.dimensions = in.readOptionalVInt();
+        this.similarity = in.readOptionalEnum(SimilarityMeasure.class);
+        this.maxInputTokens = in.readOptionalVInt();
+        this.rateLimitSettings = new RateLimitSettings(in);
+    }
+
+    public MistralEmbeddingsServiceSettings(
+        String model,
+        @Nullable Integer dimensions,
+        @Nullable Integer maxInputTokens,
+        @Nullable SimilarityMeasure similarity,
+        @Nullable RateLimitSettings rateLimitSettings
+    ) {
+        this.model = model;
+        this.dimensions = dimensions;
+        this.similarity = similarity;
+        this.maxInputTokens = maxInputTokens;
+        this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return ADD_MISTRAL_EMBEDDINGS_INFERENCE;
+    }
+
+    public String model() {
+        return this.model;
+    }
+
+    @Override
+    public Integer dimensions() {
+        return this.dimensions;
+    }
+
+    public Integer maxInputTokens() {
+        return this.maxInputTokens;
+    }
+
+    @Override
+    public SimilarityMeasure similarity() {
+        return this.similarity;
+    }
+
+    @Override
+    public DenseVectorFieldMapper.ElementType elementType() {
+        return DenseVectorFieldMapper.ElementType.FLOAT;
+    }
+
+    public RateLimitSettings rateLimitSettings() {
+        return this.rateLimitSettings;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(model);
+        out.writeOptionalVInt(dimensions);
+        out.writeOptionalEnum(SimilarityMeasure.translateSimilarity(similarity, out.getTransportVersion()));
+        out.writeOptionalVInt(maxInputTokens);
+        rateLimitSettings.writeTo(out);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        this.toXContentFragmentOfExposedFields(builder, params);
+        rateLimitSettings.toXContent(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException {
+        builder.field(MODEL_FIELD, this.model);
+
+        if (dimensions != null) {
+            builder.field(DIMENSIONS, dimensions);
+        }
+        if (similarity != null) {
+            builder.field(SIMILARITY, similarity);
+        }
+        if (this.maxInputTokens != null) {
+            builder.field(MAX_INPUT_TOKENS, this.maxInputTokens);
+        }
+
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        MistralEmbeddingsServiceSettings that = (MistralEmbeddingsServiceSettings) o;
+        return Objects.equals(model, that.model)
+            && Objects.equals(dimensions, that.dimensions)
+            && Objects.equals(maxInputTokens, that.maxInputTokens)
+            && Objects.equals(similarity, that.similarity);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(model, dimensions, maxInputTokens, similarity);
+    }
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestEntityTests.java
new file mode 100644
index 0000000000000..181ca3d5145b8
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestEntityTests.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.mistral;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.is;
+
+public class MistralEmbeddingsRequestEntityTests extends ESTestCase {
+    public void testXContent_WritesModelInputAndFormat() throws IOException {
+        var entity = new MistralEmbeddingsRequestEntity("mistral-embed", List.of("abc"));
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, is("""
+            {"model":"mistral-embed","input":["abc"],"encoding_format":"float"}"""));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestTests.java
new file mode 100644
index 0000000000000..8f78c70da0c61
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/mistral/MistralEmbeddingsRequestTests.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.request.mistral;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.HttpPost;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.common.Truncator;
+import org.elasticsearch.xpack.inference.common.TruncatorTests;
+import org.elasticsearch.xpack.inference.external.request.HttpRequest;
+import org.elasticsearch.xpack.inference.services.mistral.MistralConstants;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingModelTests;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+public class MistralEmbeddingsRequestTests extends ESTestCase {
+    public void testCreateRequest_Works() throws IOException {
+        var request = createRequest("mistral-embed", "apikey", "abcd");
+        var httpRequest = request.createHttpRequest();
+        var httpPost = validateRequestUrlAndContentType(httpRequest, MistralConstants.API_EMBEDDINGS_PATH);
+        assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer apikey"));
+
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        assertThat(requestMap, aMapWithSize(3));
+        assertThat(requestMap.get("input"), is(List.of("abcd")));
+        assertThat(requestMap.get("model"), is("mistral-embed"));
+        assertThat(requestMap.get("encoding_format"), is("float"));
+    }
+
+    public void testTruncate_ReducesInputTextSizeByHalf() throws IOException {
+        var request = createRequest("mistral-embed", "apikey", "abcd");
+        var truncatedRequest = request.truncate();
+
+        var httpRequest = truncatedRequest.createHttpRequest();
+        assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class));
+
+        var httpPost = (HttpPost) httpRequest.httpRequestBase();
+        var requestMap = entityAsMap(httpPost.getEntity().getContent());
+        assertThat(requestMap, aMapWithSize(3));
+        assertThat(requestMap.get("input"), is(List.of("ab")));
+        assertThat(requestMap.get("model"), is("mistral-embed"));
+        assertThat(requestMap.get("encoding_format"), is("float"));
+    }
+
+    public void testIsTruncated_ReturnsTrue() {
+        var request = createRequest("mistral-embed", "apikey", "abcd");
+        assertFalse(request.getTruncationInfo()[0]);
+
+        var truncatedRequest = request.truncate();
+        assertTrue(truncatedRequest.getTruncationInfo()[0]);
+    }
+
+    private HttpPost validateRequestUrlAndContentType(HttpRequest request, String expectedUrl) throws IOException {
+        assertThat(request.httpRequestBase(), instanceOf(HttpPost.class));
+        var httpPost = (HttpPost) request.httpRequestBase();
+        assertThat(httpPost.getURI().toString(), is(expectedUrl));
+        assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType()));
+        return httpPost;
+    }
+
+    public static MistralEmbeddingsRequest createRequest(String model, String apiKey, String input) {
+        var embeddingsModel = MistralEmbeddingModelTests.createModel("id", model, apiKey, null, null, null, null);
+        return new MistralEmbeddingsRequest(
+            TruncatorTests.createTruncator(),
+            new Truncator.TruncationResult(List.of(input), new boolean[] { false }),
+            embeddingsModel
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiErrorResponseEntityTests.java
index fd133a26f5532..48a560341f392 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiErrorResponseEntityTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiErrorResponseEntityTests.java
@@ -26,7 +26,7 @@ public void testErrorResponse_ExtractsError() {
         var result = getMockResult("""
             {"error":{"message":"test_error_message"}}""");
 
-        var error = AzureAndOpenAiErrorResponseEntity.fromResponse(result);
+        var error = AzureMistralOpenAiErrorResponseEntity.fromResponse(result);
         assertNotNull(error);
         assertThat(error.getErrorMessage(), is("test_error_message"));
     }
@@ -35,14 +35,14 @@ public void testErrorResponse_ReturnsNullIfNoError() {
         var result = getMockResult("""
             {"noerror":true}""");
 
-        var error = AzureAndOpenAiErrorResponseEntity.fromResponse(result);
+        var error = AzureMistralOpenAiErrorResponseEntity.fromResponse(result);
         assertNull(error);
     }
 
     public void testErrorResponse_ReturnsNullIfNotJson() {
         var result = getMockResult("not a json string");
 
-        var error = AzureAndOpenAiErrorResponseEntity.fromResponse(result);
+        var error = AzureMistralOpenAiErrorResponseEntity.fromResponse(result);
         assertNull(error);
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiExternalResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiExternalResponseHandlerTests.java
index 4c9fb143c3a5c..9ef9ab4daa0ae 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiExternalResponseHandlerTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/AzureAndOpenAiExternalResponseHandlerTests.java
@@ -42,10 +42,10 @@ public void testCheckForFailureStatusCode() {
 
         var mockRequest = RequestTests.mockRequest("id");
         var httpResult = new HttpResult(httpResponse, new byte[] {});
-        var handler = new AzureAndOpenAiExternalResponseHandler(
+        var handler = new AzureMistralOpenAiExternalResponseHandler(
             "",
             (request, result) -> null,
-            AzureAndOpenAiErrorResponseEntity::fromResponse
+            AzureMistralOpenAiErrorResponseEntity::fromResponse
         );
 
         // 200 ok
@@ -157,20 +157,20 @@ public void testBuildRateLimitErrorMessage() {
         var httpResult = new HttpResult(response, new byte[] {});
 
         {
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.REQUESTS_LIMIT)).thenReturn(
-                new BasicHeader(AzureAndOpenAiExternalResponseHandler.REQUESTS_LIMIT, "3000")
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.REQUESTS_LIMIT)).thenReturn(
+                new BasicHeader(AzureMistralOpenAiExternalResponseHandler.REQUESTS_LIMIT, "3000")
             );
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_REQUESTS)).thenReturn(
-                new BasicHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_REQUESTS, "2999")
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_REQUESTS)).thenReturn(
+                new BasicHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_REQUESTS, "2999")
             );
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.TOKENS_LIMIT)).thenReturn(
-                new BasicHeader(AzureAndOpenAiExternalResponseHandler.TOKENS_LIMIT, "10000")
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.TOKENS_LIMIT)).thenReturn(
+                new BasicHeader(AzureMistralOpenAiExternalResponseHandler.TOKENS_LIMIT, "10000")
             );
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_TOKENS)).thenReturn(
-                new BasicHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_TOKENS, "99800")
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_TOKENS)).thenReturn(
+                new BasicHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_TOKENS, "99800")
             );
 
-            var error = AzureAndOpenAiExternalResponseHandler.buildRateLimitErrorMessage(httpResult);
+            var error = AzureMistralOpenAiExternalResponseHandler.buildRateLimitErrorMessage(httpResult);
             assertThat(
                 error,
                 containsString("Token limit [10000], remaining tokens [99800]. Request limit [3000], remaining requests [2999]")
@@ -178,9 +178,9 @@ public void testBuildRateLimitErrorMessage() {
         }
 
         {
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.TOKENS_LIMIT)).thenReturn(null);
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_TOKENS)).thenReturn(null);
-            var error = AzureAndOpenAiExternalResponseHandler.buildRateLimitErrorMessage(httpResult);
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.TOKENS_LIMIT)).thenReturn(null);
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_TOKENS)).thenReturn(null);
+            var error = AzureMistralOpenAiExternalResponseHandler.buildRateLimitErrorMessage(httpResult);
             assertThat(
                 error,
                 containsString("Token limit [unknown], remaining tokens [unknown]. Request limit [3000], remaining requests [2999]")
@@ -188,26 +188,26 @@ public void testBuildRateLimitErrorMessage() {
         }
 
         {
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.REQUESTS_LIMIT)).thenReturn(null);
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_REQUESTS)).thenReturn(
-                new BasicHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_REQUESTS, "2999")
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.REQUESTS_LIMIT)).thenReturn(null);
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_REQUESTS)).thenReturn(
+                new BasicHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_REQUESTS, "2999")
             );
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.TOKENS_LIMIT)).thenReturn(null);
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_TOKENS)).thenReturn(null);
-            var error = AzureAndOpenAiExternalResponseHandler.buildRateLimitErrorMessage(httpResult);
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.TOKENS_LIMIT)).thenReturn(null);
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_TOKENS)).thenReturn(null);
+            var error = AzureMistralOpenAiExternalResponseHandler.buildRateLimitErrorMessage(httpResult);
             assertThat(error, containsString("Remaining tokens [unknown]. Remaining requests [2999]"));
         }
 
         {
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.REQUESTS_LIMIT)).thenReturn(null);
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_REQUESTS)).thenReturn(
-                new BasicHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_REQUESTS, "2999")
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.REQUESTS_LIMIT)).thenReturn(null);
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_REQUESTS)).thenReturn(
+                new BasicHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_REQUESTS, "2999")
             );
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.TOKENS_LIMIT)).thenReturn(
-                new BasicHeader(AzureAndOpenAiExternalResponseHandler.TOKENS_LIMIT, "10000")
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.TOKENS_LIMIT)).thenReturn(
+                new BasicHeader(AzureMistralOpenAiExternalResponseHandler.TOKENS_LIMIT, "10000")
             );
-            when(response.getFirstHeader(AzureAndOpenAiExternalResponseHandler.REMAINING_TOKENS)).thenReturn(null);
-            var error = AzureAndOpenAiExternalResponseHandler.buildRateLimitErrorMessage(httpResult);
+            when(response.getFirstHeader(AzureMistralOpenAiExternalResponseHandler.REMAINING_TOKENS)).thenReturn(null);
+            var error = AzureMistralOpenAiExternalResponseHandler.buildRateLimitErrorMessage(httpResult);
             assertThat(
                 error,
                 containsString("Token limit [10000], remaining tokens [unknown]. Request limit [unknown], remaining requests [2999]")
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java
new file mode 100644
index 0000000000000..3ead273e78110
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java
@@ -0,0 +1,651 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.mistral;
+
+import org.apache.http.HttpHeaders;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.inference.ChunkedInferenceServiceResults;
+import org.elasticsearch.inference.ChunkingOptions;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.inference.InputType;
+import org.elasticsearch.inference.Model;
+import org.elasticsearch.inference.ModelConfigurations;
+import org.elasticsearch.inference.ModelSecrets;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.inference.action.InferenceAction;
+import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingFloatResults;
+import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests;
+import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.logging.ThrottlerManager;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingModelTests;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsModel;
+import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettings;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.xpack.inference.Utils.getInvalidModel;
+import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool;
+import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty;
+import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
+import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
+import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
+import static org.elasticsearch.xpack.inference.services.mistral.MistralConstants.API_KEY_FIELD;
+import static org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettingsTests.createRequestSettingsMap;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class MistralServiceTests extends ESTestCase {
+    private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS);
+    private final MockWebServer webServer = new MockWebServer();
+    private ThreadPool threadPool;
+    private HttpClientManager clientManager;
+
+    @Before
+    public void init() throws Exception {
+        webServer.start();
+        threadPool = createThreadPool(inferenceUtilityPool());
+        clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class));
+    }
+
+    @After
+    public void shutdown() throws IOException {
+        clientManager.close();
+        terminate(threadPool);
+        webServer.close();
+    }
+
+    public void testParseRequestConfig_CreatesAMistralEmbeddingsModel() throws IOException {
+        try (var service = createService()) {
+            ActionListener<Model> modelVerificationListener = ActionListener.wrap(model -> {
+                assertThat(model, instanceOf(MistralEmbeddingsModel.class));
+
+                var embeddingsModel = (MistralEmbeddingsModel) model;
+                var serviceSettings = (MistralEmbeddingsServiceSettings) model.getServiceSettings();
+                assertThat(serviceSettings.model(), is("mistral-embed"));
+                assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+            }, exception -> fail("Unexpected exception: " + exception));
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.TEXT_EMBEDDING,
+                getRequestConfigMap(
+                    getEmbeddingsServiceSettingsMap("mistral-embed", null, null, null),
+                    getEmbeddingsTaskSettingsMap(),
+                    getSecretSettingsMap("secret")
+                ),
+                Set.of(),
+                modelVerificationListener
+            );
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsUnsupportedModelType() throws IOException {
+        try (var service = createService()) {
+            ActionListener<Model> modelVerificationListener = ActionListener.wrap(
+                model -> fail("Expected exception, but got model: " + model),
+                exception -> {
+                    assertThat(exception, instanceOf(ElasticsearchStatusException.class));
+                    assertThat(exception.getMessage(), is("The [mistral] service does not support task type [sparse_embedding]"));
+                }
+            );
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.SPARSE_EMBEDDING,
+                getRequestConfigMap(
+                    getEmbeddingsServiceSettingsMap("mistral-embed", null, null, null),
+                    getEmbeddingsTaskSettingsMap(),
+                    getSecretSettingsMap("secret")
+                ),
+                Set.of(),
+                modelVerificationListener
+            );
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInConfig() throws IOException {
+        try (var service = createService()) {
+            var config = getRequestConfigMap(
+                getEmbeddingsServiceSettingsMap("mistral-embed", null, null, null),
+                getEmbeddingsTaskSettingsMap(),
+                getSecretSettingsMap("secret")
+            );
+            config.put("extra_key", "value");
+
+            ActionListener<Model> modelVerificationListener = ActionListener.wrap(
+                model -> fail("Expected exception, but got model: " + model),
+                exception -> {
+                    assertThat(exception, instanceOf(ElasticsearchStatusException.class));
+                    assertThat(
+                        exception.getMessage(),
+                        is("Model configuration contains settings [{extra_key=value}] unknown to the [mistral] service")
+                    );
+                }
+            );
+
+            service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, Set.of(), modelVerificationListener);
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInEmbeddingTaskSettingsMap() throws IOException {
+        try (var service = createService()) {
+            var taskSettings = new HashMap<String, Object>();
+            taskSettings.put("extra_key", "value");
+
+            var config = getRequestConfigMap(
+                getEmbeddingsServiceSettingsMap("mistral-embed", null, null, null),
+                taskSettings,
+                getSecretSettingsMap("secret")
+            );
+
+            ActionListener<Model> modelVerificationListener = ActionListener.wrap(
+                model -> fail("Expected exception, but got model: " + model),
+                exception -> {
+                    assertThat(exception, instanceOf(ElasticsearchStatusException.class));
+                    assertThat(
+                        exception.getMessage(),
+                        is("Model configuration contains settings [{extra_key=value}] unknown to the [mistral] service")
+                    );
+                }
+            );
+
+            service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, Set.of(), modelVerificationListener);
+        }
+    }
+
+    public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInEmbeddingSecretSettingsMap() throws IOException {
+        try (var service = createService()) {
+            var secretSettings = getSecretSettingsMap("secret");
+            secretSettings.put("extra_key", "value");
+
+            var config = getRequestConfigMap(
+                getEmbeddingsServiceSettingsMap("mistral-embed", null, null, null),
+                getEmbeddingsTaskSettingsMap(),
+                secretSettings
+            );
+
+            ActionListener<Model> modelVerificationListener = ActionListener.wrap(
+                model -> fail("Expected exception, but got model: " + model),
+                exception -> {
+                    assertThat(exception, instanceOf(ElasticsearchStatusException.class));
+                    assertThat(
+                        exception.getMessage(),
+                        is("Model configuration contains settings [{extra_key=value}] unknown to the [mistral] service")
+                    );
+                }
+            );
+
+            service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, Set.of(), modelVerificationListener);
+        }
+    }
+
+    public void testParsePersistedConfig_CreatesAMistralEmbeddingsModel() throws IOException {
+        try (var service = createService()) {
+            var config = getPersistedConfigMap(
+                getEmbeddingsServiceSettingsMap("mistral-embed", 1024, 512, null),
+                getEmbeddingsTaskSettingsMap(),
+                getSecretSettingsMap("secret")
+            );
+
+            var model = service.parsePersistedConfigWithSecrets("id", TaskType.TEXT_EMBEDDING, config.config(), config.secrets());
+
+            assertThat(model, instanceOf(MistralEmbeddingsModel.class));
+
+            var embeddingsModel = (MistralEmbeddingsModel) model;
+            assertThat(embeddingsModel.getServiceSettings().model(), is("mistral-embed"));
+            assertThat(embeddingsModel.getServiceSettings().dimensions(), is(1024));
+            assertThat(embeddingsModel.getServiceSettings().maxInputTokens(), is(512));
+            assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret"));
+        }
+    }
+
+    public void testParsePersistedConfig_ThrowsUnsupportedModelType() throws IOException {
+        try (var service = createService()) {
+            ActionListener<Model> modelVerificationListener = ActionListener.wrap(
+                model -> fail("Expected exception, but got model: " + model),
+                exception -> {
+                    assertThat(exception, instanceOf(ElasticsearchStatusException.class));
+                    assertThat(exception.getMessage(), is("The [mistral] service does not support task type [sparse_embedding]"));
+                }
+            );
+
+            service.parseRequestConfig(
+                "id",
+                TaskType.SPARSE_EMBEDDING,
+                getRequestConfigMap(
+                    getEmbeddingsServiceSettingsMap("mistral-embed", null, null, null),
+                    getEmbeddingsTaskSettingsMap(),
+                    getSecretSettingsMap("secret")
+                ),
+                Set.of(),
+                modelVerificationListener
+            );
+        }
+    }
+
+    public void testParsePersistedConfigWithSecrets_ThrowsErrorTryingToParseInvalidModel() throws IOException {
+        try (var service = createService()) {
+            var config = getPersistedConfigMap(
+                getEmbeddingsServiceSettingsMap("mistral-embed", null, null, null),
+                getEmbeddingsTaskSettingsMap(),
+                getSecretSettingsMap("secret")
+            );
+
+            var thrownException = expectThrows(
+                ElasticsearchStatusException.class,
+                () -> service.parsePersistedConfigWithSecrets("id", TaskType.SPARSE_EMBEDDING, config.config(), config.secrets())
+            );
+
+            assertThat(
+                thrownException.getMessage(),
+                is("Failed to parse stored model [id] for [mistral] service, please delete and add the service again")
+            );
+        }
+    }
+
+    public void testParsePersistedConfig_DoesNotThrowWhenAnExtraKeyExistsInConfig() throws IOException {
+        try (var service = createService()) {
+            var serviceSettings = getEmbeddingsServiceSettingsMap("mistral-embed", 1024, 512, null);
+            var taskSettings = getEmbeddingsTaskSettingsMap();
+            var secretSettings = getSecretSettingsMap("secret");
+            var config = getPersistedConfigMap(serviceSettings, taskSettings, secretSettings);
+            config.config().put("extra_key", "value");
+
+            var model = service.parsePersistedConfigWithSecrets("id", TaskType.TEXT_EMBEDDING, config.config(), config.secrets());
+
+            assertThat(model, instanceOf(MistralEmbeddingsModel.class));
+        }
+    }
+
+    public void testParsePersistedConfig_DoesNotThrowWhenExtraKeyExistsInEmbeddingServiceSettingsMap() throws IOException {
+        try (var service = createService()) {
+            var serviceSettings = getEmbeddingsServiceSettingsMap("mistral-embed", 1024, 512, null);
+            serviceSettings.put("extra_key", "value");
+
+            var taskSettings = getEmbeddingsTaskSettingsMap();
+            var secretSettings = getSecretSettingsMap("secret");
+            var config = getPersistedConfigMap(serviceSettings, taskSettings, secretSettings);
+
+            var model = service.parsePersistedConfigWithSecrets("id", TaskType.TEXT_EMBEDDING, config.config(), config.secrets());
+
+            assertThat(model, instanceOf(MistralEmbeddingsModel.class));
+        }
+    }
+
+    public void testParsePersistedConfig_DoesNotThrowWhenAnExtraKeyExistsInEmbeddingTaskSettingsMap() throws IOException {
+        try (var service = createService()) {
+            var serviceSettings = getEmbeddingsServiceSettingsMap("mistral-embed", 1024, 512, null);
+            var taskSettings = new HashMap<String, Object>();
+            taskSettings.put("extra_key", "value");
+
+            var secretSettings = getSecretSettingsMap("secret");
+            var config = getPersistedConfigMap(serviceSettings, taskSettings, secretSettings);
+
+            var model = service.parsePersistedConfigWithSecrets("id", TaskType.TEXT_EMBEDDING, config.config(), config.secrets());
+
+            assertThat(model, instanceOf(MistralEmbeddingsModel.class));
+        }
+    }
+
+    public void testParsePersistedConfig_DoesNotThrowWhenAnExtraKeyExistsInEmbeddingSecretSettingsMap() throws IOException {
+        try (var service = createService()) {
+            var serviceSettings = getEmbeddingsServiceSettingsMap("mistral-embed", 1024, 512, null);
+            var taskSettings = getEmbeddingsTaskSettingsMap();
+            var secretSettings = getSecretSettingsMap("secret");
+            secretSettings.put("extra_key", "value");
+
+            var config = getPersistedConfigMap(serviceSettings, taskSettings, secretSettings);
+
+            var model = service.parsePersistedConfigWithSecrets("id", TaskType.TEXT_EMBEDDING, config.config(), config.secrets());
+
+            assertThat(model, instanceOf(MistralEmbeddingsModel.class));
+        }
+    }
+
+    public void testParsePersistedConfig_WithoutSecretsCreatesEmbeddingsModel() throws IOException {
+        try (var service = createService()) {
+            var config = getPersistedConfigMap(
+                getEmbeddingsServiceSettingsMap("mistral-embed", 1024, 512, null),
+                getEmbeddingsTaskSettingsMap(),
+                Map.of()
+            );
+
+            var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, config.config());
+
+            assertThat(model, instanceOf(MistralEmbeddingsModel.class));
+
+            var embeddingsModel = (MistralEmbeddingsModel) model;
+            assertThat(embeddingsModel.getServiceSettings().model(), is("mistral-embed"));
+            assertThat(embeddingsModel.getServiceSettings().dimensions(), is(1024));
+            assertThat(embeddingsModel.getServiceSettings().maxInputTokens(), is(512));
+        }
+    }
+
+    public void testCheckModelConfig_ForEmbeddingsModel_Works() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new MistralService(senderFactory, createWithEmptySettings(threadPool))) {
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(testEmbeddingResultJson));
+
+            var model = MistralEmbeddingModelTests.createModel("id", "mistral-embed", "apikey", null, null, null, null);
+            model.setURI(getUrl(webServer));
+
+            PlainActionFuture<Model> listener = new PlainActionFuture<>();
+            service.checkModelConfig(model, listener);
+
+            var result = listener.actionGet(TIMEOUT);
+            assertThat(
+                result,
+                is(MistralEmbeddingModelTests.createModel("id", "mistral-embed", "apikey", 2, null, SimilarityMeasure.DOT_PRODUCT, null))
+            );
+
+            assertThat(webServer.requests(), hasSize(1));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            MatcherAssert.assertThat(
+                requestMap,
+                Matchers.is(Map.of("input", List.of("how big"), "encoding_format", "float", "model", "mistral-embed"))
+            );
+        }
+    }
+
+    public void testInfer_ThrowsErrorWhenModelIsNotMistralEmbeddingsModel() throws IOException {
+        var sender = mock(Sender.class);
+
+        var factory = mock(HttpRequestSender.Factory.class);
+        when(factory.createSender(anyString())).thenReturn(sender);
+
+        var mockModel = getInvalidModel("model_id", "service_name");
+
+        try (var service = new MistralService(factory, createWithEmptySettings(threadPool))) {
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                mockModel,
+                null,
+                List.of(""),
+                new HashMap<>(),
+                InputType.INGEST,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT));
+            assertThat(
+                thrownException.getMessage(),
+                is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.")
+            );
+
+            verify(factory, times(1)).createSender(anyString());
+            verify(sender, times(1)).start();
+        }
+
+        verify(sender, times(1)).close();
+        verifyNoMoreInteractions(factory);
+        verifyNoMoreInteractions(sender);
+    }
+
+    public void testChunkedInfer_Embeddings_CallsInfer_ConvertsFloatResponse() throws IOException, URISyntaxException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new MistralService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "object": "list",
+                    "data": [
+                        {
+                            "object": "embedding",
+                            "index": 0,
+                            "embedding": [
+                                0.123,
+                                -0.123
+                            ]
+                        },
+                        {
+                            "object": "embedding",
+                            "index": 1,
+                            "embedding": [
+                                0.223,
+                                -0.223
+                            ]
+                        }
+                    ],
+                    "model": "text-embedding-ada-002-v2",
+                    "usage": {
+                        "prompt_tokens": 8,
+                        "total_tokens": 8
+                    }
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson));
+
+            var model = MistralEmbeddingModelTests.createModel("id", "mistral-embed", "apikey", null, null, null, null);
+            model.setURI(getUrl(webServer));
+
+            PlainActionFuture<List<ChunkedInferenceServiceResults>> listener = new PlainActionFuture<>();
+            service.chunkedInfer(
+                model,
+                List.of("abc", "def"),
+                new HashMap<>(),
+                InputType.INGEST,
+                new ChunkingOptions(null, null),
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var results = listener.actionGet(TIMEOUT);
+
+            assertThat(results, hasSize(2));
+            {
+                assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedTextEmbeddingFloatResults.class));
+                var floatResult = (ChunkedTextEmbeddingFloatResults) results.get(0);
+                assertThat(floatResult.chunks(), hasSize(1));
+                assertTrue(Arrays.equals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding()));
+            }
+            {
+                assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedTextEmbeddingFloatResults.class));
+                var floatResult = (ChunkedTextEmbeddingFloatResults) results.get(1);
+                assertThat(floatResult.chunks(), hasSize(1));
+                assertTrue(Arrays.equals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding()));
+            }
+
+            assertThat(webServer.requests(), hasSize(1));
+            assertNull(webServer.requests().get(0).getUri().getQuery());
+            assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType()));
+            assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer apikey"));
+
+            var requestMap = entityAsMap(webServer.requests().get(0).getBody());
+            assertThat(requestMap.size(), Matchers.is(3));
+            assertThat(requestMap.get("input"), Matchers.is(List.of("abc", "def")));
+            assertThat(requestMap.get("encoding_format"), Matchers.is("float"));
+            assertThat(requestMap.get("model"), Matchers.is("mistral-embed"));
+        }
+    }
+
+    public void testInfer_ThrowsWhenQueryIsPresent() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new MistralService(senderFactory, createWithEmptySettings(threadPool))) {
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody(testEmbeddingResultJson));
+
+            var model = MistralEmbeddingModelTests.createModel("id", "mistral-embed", "apikey", null, null, null, null);
+            model.setURI(getUrl(webServer));
+
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            UnsupportedOperationException exception = expectThrows(
+                UnsupportedOperationException.class,
+                () -> service.infer(
+                    model,
+                    "should throw",
+                    List.of("abc"),
+                    new HashMap<>(),
+                    InputType.INGEST,
+                    InferenceAction.Request.DEFAULT_TIMEOUT,
+                    listener
+                )
+            );
+
+            assertThat(exception.getMessage(), is("Mistral service does not support inference with query input"));
+        }
+    }
+
+    public void testInfer_UnauthorisedResponse() throws IOException {
+        var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
+
+        try (var service = new MistralService(senderFactory, createWithEmptySettings(threadPool))) {
+
+            String responseJson = """
+                {
+                    "error": {
+                        "message": "Incorrect API key provided:",
+                        "type": "invalid_request_error",
+                        "param": null,
+                        "code": "invalid_api_key"
+                    }
+                }
+                """;
+            webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson));
+
+            var model = MistralEmbeddingModelTests.createModel("id", "mistral-embed", "apikey", null, null, null, null);
+            model.setURI(getUrl(webServer));
+
+            PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+            service.infer(
+                model,
+                null,
+                List.of("abc"),
+                new HashMap<>(),
+                InputType.INGEST,
+                InferenceAction.Request.DEFAULT_TIMEOUT,
+                listener
+            );
+
+            var error = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT));
+            assertThat(error.getMessage(), containsString("Received an authentication error status code for request"));
+            assertThat(error.getMessage(), containsString("Error message: [Incorrect API key provided:]"));
+            assertThat(webServer.requests(), hasSize(1));
+        }
+    }
+
+    // ----------------------------------------------------------------
+
+    private MistralService createService() {
+        return new MistralService(mock(HttpRequestSender.Factory.class), createWithEmptySettings(threadPool));
+    }
+
+    private Map<String, Object> getRequestConfigMap(
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        Map<String, Object> secretSettings
+    ) {
+        var builtServiceSettings = new HashMap<>();
+        builtServiceSettings.putAll(serviceSettings);
+        builtServiceSettings.putAll(secretSettings);
+
+        return new HashMap<>(
+            Map.of(ModelConfigurations.SERVICE_SETTINGS, builtServiceSettings, ModelConfigurations.TASK_SETTINGS, taskSettings)
+        );
+    }
+
+    private record PeristedConfigRecord(Map<String, Object> config, Map<String, Object> secrets) {}
+
+    private PeristedConfigRecord getPersistedConfigMap(
+        Map<String, Object> serviceSettings,
+        Map<String, Object> taskSettings,
+        Map<String, Object> secretSettings
+    ) {
+
+        return new PeristedConfigRecord(
+            new HashMap<>(Map.of(ModelConfigurations.SERVICE_SETTINGS, serviceSettings, ModelConfigurations.TASK_SETTINGS, taskSettings)),
+            new HashMap<>(Map.of(ModelSecrets.SECRET_SETTINGS, secretSettings))
+        );
+    }
+
+    private PeristedConfigRecord getPersistedConfigMap(Map<String, Object> serviceSettings, Map<String, Object> taskSettings) {
+
+        return new PeristedConfigRecord(
+            new HashMap<>(Map.of(ModelConfigurations.SERVICE_SETTINGS, serviceSettings, ModelConfigurations.TASK_SETTINGS, taskSettings)),
+            null
+        );
+    }
+
+    private static Map<String, Object> getEmbeddingsServiceSettingsMap(
+        String model,
+        @Nullable Integer dimensions,
+        @Nullable Integer maxTokens,
+        @Nullable SimilarityMeasure similarityMeasure
+    ) {
+        return createRequestSettingsMap(model, dimensions, maxTokens, similarityMeasure);
+    }
+
+    private static Map<String, Object> getEmbeddingsTaskSettingsMap() {
+        // no task settings for Mistral embeddings
+        return Map.of();
+    }
+
+    private static Map<String, Object> getSecretSettingsMap(String apiKey) {
+        return new HashMap<>(Map.of(API_KEY_FIELD, apiKey));
+    }
+
+    private static final String testEmbeddingResultJson = """
+        {
+          "object": "list",
+          "data": [
+              {
+                  "object": "embedding",
+                  "index": 0,
+                  "embedding": [
+                      0.0123,
+                      -0.0123
+                  ]
+              }
+          ],
+          "model": "text-embedding-ada-002-v2",
+          "usage": {
+              "prompt_tokens": 8,
+              "total_tokens": 8
+          }
+        }
+        """;
+
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingModelTests.java
new file mode 100644
index 0000000000000..0fe8723664c6e
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingModelTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.mistral.embeddings;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.EmptyTaskSettings;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.inference.TaskType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+public class MistralEmbeddingModelTests extends ESTestCase {
+    public static MistralEmbeddingsModel createModel(String inferenceId, String model, String apiKey) {
+        return createModel(inferenceId, model, apiKey, null, null, null, null);
+    }
+
+    public static MistralEmbeddingsModel createModel(
+        String inferenceId,
+        String model,
+        String apiKey,
+        @Nullable Integer dimensions,
+        @Nullable Integer maxTokens,
+        @Nullable SimilarityMeasure similarity,
+        RateLimitSettings rateLimitSettings
+    ) {
+        return new MistralEmbeddingsModel(
+            inferenceId,
+            TaskType.TEXT_EMBEDDING,
+            "mistral",
+            new MistralEmbeddingsServiceSettings(model, dimensions, maxTokens, similarity, rateLimitSettings),
+            EmptyTaskSettings.INSTANCE,
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
+        );
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettingsTests.java
new file mode 100644
index 0000000000000..13f43a5f31ad3
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettingsTests.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.services.mistral.embeddings;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.ByteArrayStreamInput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.ServiceFields;
+import org.elasticsearch.xpack.inference.services.mistral.MistralConstants;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+import org.hamcrest.CoreMatchers;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY;
+import static org.hamcrest.Matchers.is;
+
+public class MistralEmbeddingsServiceSettingsTests extends ESTestCase {
+    public void testFromMap_Request_CreatesSettingsCorrectly() {
+        var model = "mistral-embed";
+        var dims = 1536;
+        var maxInputTokens = 512;
+        var serviceSettings = MistralEmbeddingsServiceSettings.fromMap(
+            createRequestSettingsMap(model, dims, maxInputTokens, SimilarityMeasure.COSINE),
+            ConfigurationParseContext.REQUEST
+        );
+
+        assertThat(serviceSettings, is(new MistralEmbeddingsServiceSettings(model, dims, maxInputTokens, SimilarityMeasure.COSINE, null)));
+    }
+
+    public void testFromMap_RequestWithRateLimit_CreatesSettingsCorrectly() {
+        var model = "mistral-embed";
+        var dims = 1536;
+        var maxInputTokens = 512;
+        var settingsMap = createRequestSettingsMap(model, dims, maxInputTokens, SimilarityMeasure.COSINE);
+        settingsMap.put(RateLimitSettings.FIELD_NAME, new HashMap<>(Map.of(RateLimitSettings.REQUESTS_PER_MINUTE_FIELD, 3)));
+
+        var serviceSettings = MistralEmbeddingsServiceSettings.fromMap(settingsMap, ConfigurationParseContext.REQUEST);
+
+        assertThat(
+            serviceSettings,
+            is(new MistralEmbeddingsServiceSettings(model, dims, maxInputTokens, SimilarityMeasure.COSINE, new RateLimitSettings(3)))
+        );
+    }
+
+    public void testFromMap_Persistent_CreatesSettingsCorrectly() {
+        var model = "mistral-embed";
+        var dims = 1536;
+        var maxInputTokens = 512;
+
+        var settingsMap = createRequestSettingsMap(model, dims, maxInputTokens, SimilarityMeasure.COSINE);
+        var serviceSettings = MistralEmbeddingsServiceSettings.fromMap(settingsMap, ConfigurationParseContext.PERSISTENT);
+
+        assertThat(serviceSettings, is(new MistralEmbeddingsServiceSettings(model, dims, maxInputTokens, SimilarityMeasure.COSINE, null)));
+    }
+
+    public void testFromMap_PersistentContext_DoesNotThrowException_WhenDimensionsIsNull() {
+        var model = "mistral-embed";
+
+        var settingsMap = createRequestSettingsMap(model, null, null, null);
+        var serviceSettings = MistralEmbeddingsServiceSettings.fromMap(settingsMap, ConfigurationParseContext.PERSISTENT);
+
+        assertThat(serviceSettings, is(new MistralEmbeddingsServiceSettings(model, null, null, null, null)));
+    }
+
+    public void testFromMap_PersistentContext_DoesNotThrowException_WhenSimilarityIsPresent() {
+        var model = "mistral-embed";
+
+        var settingsMap = createRequestSettingsMap(model, null, null, SimilarityMeasure.DOT_PRODUCT);
+        var serviceSettings = MistralEmbeddingsServiceSettings.fromMap(settingsMap, ConfigurationParseContext.PERSISTENT);
+
+        assertThat(serviceSettings, is(new MistralEmbeddingsServiceSettings(model, null, null, SimilarityMeasure.DOT_PRODUCT, null)));
+    }
+
+    public void testToXContent_WritesAllValues() throws IOException {
+        var entity = new MistralEmbeddingsServiceSettings("model_name", 1024, 512, null, new RateLimitSettings(3));
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        entity.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, CoreMatchers.is("""
+            {"model":"model_name","dimensions":1024,"max_input_tokens":512,""" + """
+            "rate_limit":{"requests_per_minute":3}}"""));
+    }
+
+    public void testToFilteredXContent_WritesFilteredValues() throws IOException {
+        var entity = new MistralEmbeddingsServiceSettings("model_name", 1024, 512, null, new RateLimitSettings(3));
+
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        var filteredXContent = entity.getFilteredXContentObject();
+        filteredXContent.toXContent(builder, null);
+        String xContentResult = Strings.toString(builder);
+
+        assertThat(xContentResult, CoreMatchers.is("""
+            {"model":"model_name","dimensions":1024,"max_input_tokens":512}"""));
+    }
+
+    public void testStreamInputAndOutput_WritesValuesCorrectly() throws IOException {
+        var outputBuffer = new BytesStreamOutput();
+        var settings = new MistralEmbeddingsServiceSettings("model_name", 1024, 512, null, new RateLimitSettings(3));
+        settings.writeTo(outputBuffer);
+
+        var outputBufferRef = outputBuffer.bytes();
+        var inputBuffer = new ByteArrayStreamInput(outputBufferRef.array());
+
+        var settingsFromBuffer = new MistralEmbeddingsServiceSettings(inputBuffer);
+
+        assertEquals(settings, settingsFromBuffer);
+    }
+
+    public static HashMap<String, Object> createRequestSettingsMap(
+        String model,
+        @Nullable Integer dimensions,
+        @Nullable Integer maxTokens,
+        @Nullable SimilarityMeasure similarityMeasure
+    ) {
+        var map = new HashMap<String, Object>(Map.of(MistralConstants.MODEL_FIELD, model));
+
+        if (dimensions != null) {
+            map.put(ServiceFields.DIMENSIONS, dimensions);
+        }
+
+        if (maxTokens != null) {
+            map.put(ServiceFields.MAX_INPUT_TOKENS, maxTokens);
+        }
+
+        if (similarityMeasure != null) {
+            map.put(SIMILARITY, similarityMeasure.toString());
+        }
+
+        return map;
+    }
+
+}

From 804941978545e95ba3444274781d919677118fb7 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 3 Jun 2024 10:59:22 -0400
Subject: [PATCH 111/208] ESQL: Move serialization of `Attribute` (#109267)

This moves the serialization of `Attribute` classes used in ESQL into
the classes themselves to better line up with the rest of Elasticsearch.
---
 .../xpack/esql/core/expression/Attribute.java |   8 +-
 .../esql/core/expression/EmptyAttribute.java  |  14 +-
 .../esql/core/expression/FieldAttribute.java  |  51 +++++
 .../core/expression/MetadataAttribute.java    |  64 ++++++
 .../xpack/esql/core/expression/NameId.java    |   5 +
 .../core/expression/ReferenceAttribute.java   |  48 +++++
 .../core/expression/UnresolvedAttribute.java  |  13 +-
 .../xpack/esql/core/tree/Source.java          |   5 +
 .../core/type/AbstractEsFieldTypeTests.java   |  24 ++-
 .../esql/core/type/DateEsFieldTests.java      |   3 +-
 .../xpack/esql/core/type/EsFieldTests.java    |   3 +-
 .../core/type/InvalidMappedFieldTests.java    |   3 +-
 .../esql/core/type/KeywordEsFieldTests.java   |   3 +-
 .../esql/core/type/TextEsFieldTests.java      |   3 +-
 .../core/type/UnsupportedEsFieldTests.java    |   5 +-
 .../function/UnsupportedAttribute.java        |  36 +++-
 .../xpack/esql/io/stream/PlanNamedTypes.java  | 198 ++++++------------
 .../xpack/esql/io/stream/PlanStreamInput.java |  21 --
 .../esql/io/stream/PlanStreamOutput.java      |   5 -
 .../xpack/esql/plugin/EsqlPlugin.java         |   4 +
 .../xpack/esql/SerializationTestUtils.java    |   4 +
 .../function/AbstractAttributeTestCase.java   | 123 +++++++++++
 .../function/FieldAttributeTests.java         |  58 +++++
 .../function/MetadataAttributeTests.java      |  49 +++++
 .../function/ReferenceAttributeTests.java     |  46 ++++
 .../function/UnsupportedAttributeTests.java   |  39 ++++
 .../esql/io/stream/PlanNamedTypesTests.java   |  52 -----
 .../esql/plugin/DataNodeRequestTests.java     |   4 +-
 28 files changed, 649 insertions(+), 242 deletions(-)
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAttributeTestCase.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttributeTests.java

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
index 0736ad3d1e296..5326825ec1105 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
@@ -6,6 +6,8 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -28,7 +30,11 @@
  * is a named expression (an {@code Alias} will be created automatically for it).
  * The rest are not as they are not part of the projection and thus are not part of the derived table.
  */
-public abstract class Attribute extends NamedExpression {
+public abstract class Attribute extends NamedExpression implements NamedWriteable {
+    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        // TODO add UnsupportedAttribute when these are moved to the same project
+        return List.of(FieldAttribute.ENTRY, MetadataAttribute.ENTRY, ReferenceAttribute.ENTRY);
+    }
 
     // empty - such as a top level attribute in SELECT cause
     // present - table name or a table name alias
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java
index 56e5c65b179f6..7a724eaa2be65 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java
@@ -7,22 +7,34 @@
 
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
+import java.io.IOException;
+
 /**
  * Marker for optional attributes. Acting as a dummy placeholder to avoid using null
  * in the tree (which is not allowed).
  */
 public class EmptyAttribute extends Attribute {
-
     public EmptyAttribute(Source source) {
         super(source, StringUtils.EMPTY, null, null);
     }
 
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        throw new UnsupportedOperationException("doesn't escape the node");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("doesn't escape the node");
+    }
+
     @Override
     protected Attribute clone(
         Source source,
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
index 9ee9732b542a0..35fe402035f69 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
@@ -7,12 +7,17 @@
 package org.elasticsearch.xpack.esql.core.expression;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
+import java.io.IOException;
 import java.util.Objects;
 
 /**
@@ -24,6 +29,11 @@
  * - nestedParent - if nested, what's the parent (which might not be the immediate one)
  */
 public class FieldAttribute extends TypedAttribute {
+    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        Attribute.class,
+        "FieldAttribute",
+        FieldAttribute::new
+    );
 
     private final FieldAttribute parent;
     private final String path;
@@ -67,6 +77,47 @@ public FieldAttribute(
         this.field = field;
     }
 
+    @SuppressWarnings("unchecked")
+    public <S extends StreamInput & PlanStreamInput> FieldAttribute(StreamInput in) throws IOException {
+        /*
+         * The funny casting dance with `<S extends...>` and `(S) in` is required
+         * because we're in esql-core here and the real PlanStreamInput is in
+         * esql-proper. And because NamedWriteableRegistry.Entry needs StreamInput,
+         * not a PlanStreamInput. And we need PlanStreamInput to handle Source
+         * and NameId. This should become a hard cast when we move everything out
+         * of esql-core.
+         */
+        this(
+            Source.readFrom((S) in),
+            in.readOptionalWriteable(FieldAttribute::new),
+            in.readString(),
+            DataType.readFrom(in),
+            in.readNamedWriteable(EsField.class),
+            in.readOptionalString(),
+            in.readEnum(Nullability.class),
+            NameId.readFrom((S) in),
+            in.readBoolean()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeOptionalWriteable(parent);
+        out.writeString(name());
+        dataType().writeTo(out);
+        out.writeNamedWriteable(field);
+        out.writeOptionalString(qualifier());
+        out.writeEnum(nullable());
+        id().writeTo(out);
+        out.writeBoolean(synthetic());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<FieldAttribute> info() {
         return NodeInfo.create(this, FieldAttribute::new, parent, name(), dataType(), field, qualifier(), nullable(), id(), synthetic());
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
index e6777a9ab4bb4..9cbee26f443ba 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
@@ -7,6 +7,9 @@
 
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
@@ -15,12 +18,20 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
 
+import java.io.IOException;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.core.Tuple.tuple;
 
 public class MetadataAttribute extends TypedAttribute {
+    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        Attribute.class,
+        "MetadataAttribute",
+        MetadataAttribute::new
+    );
 
     private static final Map<String, Tuple<DataType, Boolean>> ATTRIBUTES_MAP = Map.of(
         "_version",
@@ -55,6 +66,45 @@ public MetadataAttribute(Source source, String name, DataType dataType, boolean
         this(source, name, dataType, null, Nullability.TRUE, null, false, searchable);
     }
 
+    @SuppressWarnings("unchecked")
+    public <S extends StreamInput & PlanStreamInput> MetadataAttribute(StreamInput in) throws IOException {
+        /*
+         * The funny casting dance with `<S extends...>` and `(S) in` is required
+         * because we're in esql-core here and the real PlanStreamInput is in
+         * esql-proper. And because NamedWriteableRegistry.Entry needs StreamInput,
+         * not a PlanStreamInput. And we need PlanStreamInput to handle Source
+         * and NameId. This should become a hard cast when we move everything out
+         * of esql-core.
+         */
+        this(
+            Source.readFrom((S) in),
+            in.readString(),
+            DataType.readFrom(in),
+            in.readOptionalString(),
+            in.readEnum(Nullability.class),
+            NameId.readFrom((S) in),
+            in.readBoolean(),
+            in.readBoolean()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeString(name());
+        dataType().writeTo(out);
+        out.writeOptionalString(qualifier());
+        out.writeEnum(nullable());
+        id().writeTo(out);
+        out.writeBoolean(synthetic());
+        out.writeBoolean(searchable);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected MetadataAttribute clone(
         Source source,
@@ -99,4 +149,18 @@ public static DataType dataType(String name) {
     public static boolean isSupported(String name) {
         return ATTRIBUTES_MAP.containsKey(name);
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (false == super.equals(obj)) {
+            return false;
+        }
+        MetadataAttribute other = (MetadataAttribute) obj;
+        return searchable == other.searchable;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), searchable);
+    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java
index 2aa70397075d0..d2d01857a1f73 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java
@@ -55,6 +55,11 @@ public String toString() {
     }
 
     public static <S extends StreamInput & PlanStreamInput> NameId readFrom(S in) throws IOException {
+        /*
+         * The funny typing dance with `<S extends...>` is required we're in esql-core
+         * here and the real PlanStreamInput is in esql-proper. And we need PlanStreamInput
+         * to properly map NameIds.
+         */
         long unmappedId = in.readLong();
         return in.mapNameId(unmappedId);
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
index d9311dfa27edd..8bac20e9347bc 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
@@ -6,14 +6,25 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
+
+import java.io.IOException;
 
 /**
  * Attribute based on a reference to an expression.
  */
 public class ReferenceAttribute extends TypedAttribute {
+    static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        Attribute.class,
+        "ReferenceAttribute",
+        ReferenceAttribute::new
+    );
 
     public ReferenceAttribute(Source source, String name, DataType dataType) {
         this(source, name, dataType, null, Nullability.FALSE, null, false);
@@ -31,6 +42,43 @@ public ReferenceAttribute(
         super(source, name, dataType, qualifier, nullability, id, synthetic);
     }
 
+    @SuppressWarnings("unchecked")
+    public <S extends StreamInput & PlanStreamInput> ReferenceAttribute(StreamInput in) throws IOException {
+        /*
+         * The funny casting dance with `<S extends...>` and `(S) in` is required
+         * because we're in esql-core here and the real PlanStreamInput is in
+         * esql-proper. And because NamedWriteableRegistry.Entry needs StreamInput,
+         * not a PlanStreamInput. And we need PlanStreamInput to handle Source
+         * and NameId. This should become a hard cast when we move everything out
+         * of esql-core.
+         */
+        this(
+            Source.readFrom((S) in),
+            in.readString(),
+            DataType.readFrom(in),
+            in.readOptionalString(),
+            in.readEnum(Nullability.class),
+            NameId.readFrom((S) in),
+            in.readBoolean()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeString(name());
+        dataType().writeTo(out);
+        out.writeOptionalString(qualifier());
+        out.writeEnum(nullable());
+        id().writeTo(out);
+        out.writeBoolean(synthetic());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected Attribute clone(
         Source source,
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java
index 923a72d311166..87ef37cb84d1f 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
 import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
@@ -13,12 +14,12 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 // unfortunately we can't use UnresolvedNamedExpression
 public class UnresolvedAttribute extends Attribute implements Unresolvable {
-
     private final String unresolvedMsg;
     private final boolean customMessage;
     private final Object resolutionMetadata;
@@ -50,6 +51,16 @@ public UnresolvedAttribute(
         this.resolutionMetadata = resolutionMetadata;
     }
 
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        throw new UnsupportedOperationException("doesn't escape the node");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("doesn't escape the node");
+    }
+
     @Override
     protected NodeInfo<UnresolvedAttribute> info() {
         return NodeInfo.create(this, UnresolvedAttribute::new, name(), qualifier(), id(), unresolvedMsg, resolutionMetadata);
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java
index 2129bd8743d0f..e53593e944632 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java
@@ -34,6 +34,11 @@ public Source(Location location, String text) {
     }
 
     public static <S extends StreamInput & PlanStreamInput> Source readFrom(S in) throws IOException {
+        /*
+         * The funny typing dance with `<S extends...>` is required we're in esql-core
+         * here and the real PlanStreamInput is in esql-proper. And we need PlanStreamInput
+         * to send the query one time.
+         */
         if (in.readBoolean() == false) {
             return EMPTY;
         }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java
index 372bd7ec8066a..a415c529894c3 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/AbstractEsFieldTypeTests.java
@@ -15,10 +15,22 @@
 import java.util.TreeMap;
 
 public abstract class AbstractEsFieldTypeTests<T extends EsField> extends AbstractNamedWriteableTestCase<EsField> {
+    public static EsField randomAnyEsField(int maxDepth) {
+        return switch (between(0, 5)) {
+            case 0 -> EsFieldTests.randomEsField(maxDepth);
+            case 1 -> DateEsFieldTests.randomDateEsField(maxDepth);
+            case 2 -> InvalidMappedFieldTests.randomInvalidMappedField(maxDepth);
+            case 3 -> KeywordEsFieldTests.randomKeywordEsField(maxDepth);
+            case 4 -> TextEsFieldTests.randomTextEsField(maxDepth);
+            case 5 -> UnsupportedEsFieldTests.randomUnsupportedEsField(maxDepth);
+            default -> throw new IllegalArgumentException();
+        };
+    }
+
     @Override
     protected abstract T createTestInstance();
 
-    protected abstract T mutate(T instance) throws IOException;
+    protected abstract T mutate(T instance);
 
     /**
      * Generate sub-properties.
@@ -34,15 +46,7 @@ static Map<String, EsField> randomProperties(int maxDepth) {
         int targetSize = between(1, 5);
         Map<String, EsField> properties = new TreeMap<>();
         while (properties.size() < targetSize) {
-            properties.put(randomAlphaOfLength(properties.size() + 1), switch (between(0, 5)) {
-                case 0 -> EsFieldTests.randomEsField(maxDepth - 1);
-                case 1 -> DateEsFieldTests.randomDateEsField(maxDepth - 1);
-                case 2 -> InvalidMappedFieldTests.randomInvalidMappedField(maxDepth - 1);
-                case 3 -> KeywordEsFieldTests.randomKeywordEsField(maxDepth - 1);
-                case 4 -> TextEsFieldTests.randomTextEsField(maxDepth - 1);
-                case 5 -> UnsupportedEsFieldTests.randomUnsupportedEsField(maxDepth - 1);
-                default -> throw new IllegalArgumentException();
-            });
+            properties.put(randomAlphaOfLength(properties.size() + 1), randomAnyEsField(maxDepth - 1));
         }
         return properties;
     }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java
index c6428034eaae1..dea03ee8a8cdf 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DateEsFieldTests.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.esql.core.type;
 
-import java.io.IOException;
 import java.util.Map;
 
 public class DateEsFieldTests extends AbstractEsFieldTypeTests<DateEsField> {
@@ -21,7 +20,7 @@ protected DateEsField createTestInstance() {
     }
 
     @Override
-    protected DateEsField mutate(DateEsField instance) throws IOException {
+    protected DateEsField mutate(DateEsField instance) {
         String name = instance.getName();
         Map<String, EsField> properties = instance.getProperties();
         boolean aggregatable = instance.isAggregatable();
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
index 2d75def3296d3..75921778d5970 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.esql.core.type;
 
-import java.io.IOException;
 import java.util.Map;
 
 public class EsFieldTests extends AbstractEsFieldTypeTests<EsField> {
@@ -26,7 +25,7 @@ protected EsField createTestInstance() {
     }
 
     @Override
-    protected EsField mutate(EsField instance) throws IOException {
+    protected EsField mutate(EsField instance) {
         String name = instance.getName();
         DataType esDataType = instance.getDataType();
         Map<String, EsField> properties = instance.getProperties();
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java
index 5f96b4a720381..47a99329d0222 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedFieldTests.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.esql.core.type;
 
-import java.io.IOException;
 import java.util.Map;
 
 public class InvalidMappedFieldTests extends AbstractEsFieldTypeTests<InvalidMappedField> {
@@ -24,7 +23,7 @@ protected InvalidMappedField createTestInstance() {
     }
 
     @Override
-    protected InvalidMappedField mutate(InvalidMappedField instance) throws IOException {
+    protected InvalidMappedField mutate(InvalidMappedField instance) {
         String name = instance.getName();
         String errorMessage = instance.errorMessage();
         Map<String, EsField> properties = instance.getProperties();
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java
index 0b1c866114737..a5d3b8329b2df 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/KeywordEsFieldTests.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
 import java.util.Map;
 
 public class KeywordEsFieldTests extends AbstractEsFieldTypeTests<KeywordEsField> {
@@ -29,7 +28,7 @@ protected KeywordEsField createTestInstance() {
     }
 
     @Override
-    protected KeywordEsField mutate(KeywordEsField instance) throws IOException {
+    protected KeywordEsField mutate(KeywordEsField instance) {
         String name = instance.getName();
         Map<String, EsField> properties = instance.getProperties();
         boolean hasDocValues = instance.isAggregatable();
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java
index df00bf27c32b9..817dd7cd27094 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TextEsFieldTests.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.esql.core.type;
 
-import java.io.IOException;
 import java.util.Map;
 
 public class TextEsFieldTests extends AbstractEsFieldTypeTests<TextEsField> {
@@ -25,7 +24,7 @@ protected TextEsField createTestInstance() {
     }
 
     @Override
-    protected TextEsField mutate(TextEsField instance) throws IOException {
+    protected TextEsField mutate(TextEsField instance) {
         String name = instance.getName();
         Map<String, EsField> properties = instance.getProperties();
         boolean hasDocValues = instance.isAggregatable();
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java
index fb1ca014a0ee0..e05d8ca10425e 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsFieldTests.java
@@ -7,11 +7,10 @@
 
 package org.elasticsearch.xpack.esql.core.type;
 
-import java.io.IOException;
 import java.util.Map;
 
 public class UnsupportedEsFieldTests extends AbstractEsFieldTypeTests<UnsupportedEsField> {
-    static UnsupportedEsField randomUnsupportedEsField(int maxPropertiesDepth) {
+    public static UnsupportedEsField randomUnsupportedEsField(int maxPropertiesDepth) {
         String name = randomAlphaOfLength(4);
         String originalType = randomAlphaOfLength(5);
         String inherited = randomBoolean() ? null : randomAlphaOfLength(5);
@@ -25,7 +24,7 @@ protected UnsupportedEsField createTestInstance() {
     }
 
     @Override
-    protected UnsupportedEsField mutate(UnsupportedEsField instance) throws IOException {
+    protected UnsupportedEsField mutate(UnsupportedEsField instance) {
         String name = instance.getName();
         String originalType = randomAlphaOfLength(5);
         String inherited = randomBoolean() ? null : randomAlphaOfLength(5);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
index 9c37dab90db54..fe6db916f7a0d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
@@ -7,6 +7,9 @@
 
 package org.elasticsearch.xpack.esql.expression.function;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
@@ -16,7 +19,9 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
+import java.io.IOException;
 import java.util.Objects;
 
 /**
@@ -26,9 +31,14 @@
  * As such the field is marked as unresolved (so the verifier can pick up its usage outside project).
  */
 public final class UnsupportedAttribute extends FieldAttribute implements Unresolvable {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+        Attribute.class,
+        "UnsupportedAttribute",
+        UnsupportedAttribute::new
+    );
 
     private final String message;
-    private final boolean hasCustomMessage;
+    private final boolean hasCustomMessage; // TODO remove me and just use message != null?
 
     private static String errorMessage(String name, UnsupportedEsField field) {
         return "Cannot use field [" + name + "] with unsupported type [" + field.getOriginalType() + "]";
@@ -48,6 +58,30 @@ public UnsupportedAttribute(Source source, String name, UnsupportedEsField field
         this.message = customMessage == null ? errorMessage(qualifiedName(), field) : customMessage;
     }
 
+    public UnsupportedAttribute(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((PlanStreamInput) in),
+            in.readString(),
+            new UnsupportedEsField(in),
+            in.readOptionalString(),
+            NameId.readFrom((PlanStreamInput) in)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeString(name());
+        field().writeTo(out);
+        out.writeOptionalString(hasCustomMessage ? message : null);
+        id().writeTo(out);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     public boolean resolved() {
         return false;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index c0859782937bc..f605f898366e1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
-import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
@@ -51,7 +50,6 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
-import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Avg;
@@ -299,10 +297,10 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(LogicalPlan.class, Project.class, PlanNamedTypes::writeProject, PlanNamedTypes::readProject),
             of(LogicalPlan.class, TopN.class, PlanNamedTypes::writeTopN, PlanNamedTypes::readTopN),
             // Attributes
-            of(Attribute.class, FieldAttribute.class, PlanNamedTypes::writeFieldAttribute, PlanNamedTypes::readFieldAttribute),
-            of(Attribute.class, ReferenceAttribute.class, PlanNamedTypes::writeReferenceAttr, PlanNamedTypes::readReferenceAttr),
-            of(Attribute.class, MetadataAttribute.class, PlanNamedTypes::writeMetadataAttr, PlanNamedTypes::readMetadataAttr),
-            of(Attribute.class, UnsupportedAttribute.class, PlanNamedTypes::writeUnsupportedAttr, PlanNamedTypes::readUnsupportedAttr),
+            of(NamedExpression.class, FieldAttribute.class, (o, a) -> a.writeTo(o), FieldAttribute::new),
+            of(NamedExpression.class, ReferenceAttribute.class, (o, a) -> a.writeTo(o), ReferenceAttribute::new),
+            of(NamedExpression.class, MetadataAttribute.class, (o, a) -> a.writeTo(o), MetadataAttribute::new),
+            of(NamedExpression.class, UnsupportedAttribute.class, (o, a) -> a.writeTo(o), UnsupportedAttribute::new),
             // NamedExpressions
             of(NamedExpression.class, Alias.class, PlanNamedTypes::writeAlias, PlanNamedTypes::readAlias),
             // BinaryComparison
@@ -473,7 +471,7 @@ static DissectExec readDissectExec(PlanStreamInput in) throws IOException {
             in.readPhysicalPlanNode(),
             in.readExpression(),
             readDissectParser(in),
-            readAttributes(in)
+            in.readNamedWriteableCollectionAsList(Attribute.class)
         );
     }
 
@@ -482,7 +480,7 @@ static void writeDissectExec(PlanStreamOutput out, DissectExec dissectExec) thro
         out.writePhysicalPlanNode(dissectExec.child());
         out.writeExpression(dissectExec.inputExpression());
         writeDissectParser(out, dissectExec.parser());
-        writeAttributes(out, dissectExec.extractedFields());
+        out.writeNamedWriteableCollection(dissectExec.extractedFields());
     }
 
     static EsQueryExec readEsQueryExec(PlanStreamInput in) throws IOException {
@@ -490,7 +488,7 @@ static EsQueryExec readEsQueryExec(PlanStreamInput in) throws IOException {
             Source.readFrom(in),
             readEsIndex(in),
             readIndexMode(in),
-            readAttributes(in),
+            in.readNamedWriteableCollectionAsList(Attribute.class),
             in.readOptionalNamedWriteable(QueryBuilder.class),
             in.readOptionalNamed(Expression.class),
             in.readOptionalCollectionAsList(readerFromPlanReader(PlanNamedTypes::readFieldSort)),
@@ -503,7 +501,7 @@ static void writeEsQueryExec(PlanStreamOutput out, EsQueryExec esQueryExec) thro
         Source.EMPTY.writeTo(out);
         writeEsIndex(out, esQueryExec.index());
         writeIndexMode(out, esQueryExec.indexMode());
-        writeAttributes(out, esQueryExec.output());
+        out.writeNamedWriteableCollection(esQueryExec.output());
         out.writeOptionalNamedWriteable(esQueryExec.query());
         out.writeOptionalExpression(esQueryExec.limit());
         out.writeOptionalCollection(esQueryExec.sorts(), writerFromPlanWriter(PlanNamedTypes::writeFieldSort));
@@ -514,7 +512,7 @@ static EsSourceExec readEsSourceExec(PlanStreamInput in) throws IOException {
         return new EsSourceExec(
             Source.readFrom(in),
             readEsIndex(in),
-            readAttributes(in),
+            in.readNamedWriteableCollectionAsList(Attribute.class),
             in.readOptionalNamedWriteable(QueryBuilder.class),
             readIndexMode(in)
         );
@@ -523,7 +521,7 @@ static EsSourceExec readEsSourceExec(PlanStreamInput in) throws IOException {
     static void writeEsSourceExec(PlanStreamOutput out, EsSourceExec esSourceExec) throws IOException {
         Source.EMPTY.writeTo(out);
         writeEsIndex(out, esSourceExec.index());
-        writeAttributes(out, esSourceExec.output());
+        out.writeNamedWriteableCollection(esSourceExec.output());
         out.writeOptionalNamedWriteable(esSourceExec.query());
         writeIndexMode(out, esSourceExec.indexMode());
     }
@@ -613,44 +611,54 @@ static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOEx
     }
 
     static ExchangeExec readExchangeExec(PlanStreamInput in) throws IOException {
-        return new ExchangeExec(Source.readFrom(in), readAttributes(in), in.readBoolean(), in.readPhysicalPlanNode());
+        return new ExchangeExec(
+            Source.readFrom(in),
+            in.readNamedWriteableCollectionAsList(Attribute.class),
+            in.readBoolean(),
+            in.readPhysicalPlanNode()
+        );
     }
 
     static void writeExchangeExec(PlanStreamOutput out, ExchangeExec exchangeExec) throws IOException {
         Source.EMPTY.writeTo(out);
-        writeAttributes(out, exchangeExec.output());
+        out.writeNamedWriteableCollection(exchangeExec.output());
         out.writeBoolean(exchangeExec.isInBetweenAggs());
         out.writePhysicalPlanNode(exchangeExec.child());
     }
 
     static ExchangeSinkExec readExchangeSinkExec(PlanStreamInput in) throws IOException {
-        return new ExchangeSinkExec(Source.readFrom(in), readAttributes(in), in.readBoolean(), in.readPhysicalPlanNode());
+        return new ExchangeSinkExec(
+            Source.readFrom(in),
+            in.readNamedWriteableCollectionAsList(Attribute.class),
+            in.readBoolean(),
+            in.readPhysicalPlanNode()
+        );
     }
 
     static void writeExchangeSinkExec(PlanStreamOutput out, ExchangeSinkExec exchangeSinkExec) throws IOException {
         Source.EMPTY.writeTo(out);
-        writeAttributes(out, exchangeSinkExec.output());
+        out.writeNamedWriteableCollection(exchangeSinkExec.output());
         out.writeBoolean(exchangeSinkExec.isIntermediateAgg());
         out.writePhysicalPlanNode(exchangeSinkExec.child());
     }
 
     static ExchangeSourceExec readExchangeSourceExec(PlanStreamInput in) throws IOException {
-        return new ExchangeSourceExec(Source.readFrom(in), readAttributes(in), in.readBoolean());
+        return new ExchangeSourceExec(Source.readFrom(in), in.readNamedWriteableCollectionAsList(Attribute.class), in.readBoolean());
     }
 
     static void writeExchangeSourceExec(PlanStreamOutput out, ExchangeSourceExec exchangeSourceExec) throws IOException {
-        writeAttributes(out, exchangeSourceExec.output());
+        out.writeNamedWriteableCollection(exchangeSourceExec.output());
         out.writeBoolean(exchangeSourceExec.isIntermediateAgg());
     }
 
     static FieldExtractExec readFieldExtractExec(PlanStreamInput in) throws IOException {
-        return new FieldExtractExec(Source.readFrom(in), in.readPhysicalPlanNode(), readAttributes(in));
+        return new FieldExtractExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readNamedWriteableCollectionAsList(Attribute.class));
     }
 
     static void writeFieldExtractExec(PlanStreamOutput out, FieldExtractExec fieldExtractExec) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(fieldExtractExec.child());
-        writeAttributes(out, fieldExtractExec.attributesToExtract());
+        out.writeNamedWriteableCollection(fieldExtractExec.attributesToExtract());
     }
 
     static FilterExec readFilterExec(PlanStreamInput in) throws IOException {
@@ -690,7 +698,7 @@ static GrokExec readGrokExec(PlanStreamInput in) throws IOException {
             in.readPhysicalPlanNode(),
             in.readExpression(),
             Grok.pattern(source, in.readString()),
-            readAttributes(in)
+            in.readNamedWriteableCollectionAsList(Attribute.class)
         );
     }
 
@@ -699,7 +707,7 @@ static void writeGrokExec(PlanStreamOutput out, GrokExec grokExec) throws IOExce
         out.writePhysicalPlanNode(grokExec.child());
         out.writeExpression(grokExec.inputExpression());
         out.writeString(grokExec.pattern().pattern());
-        writeAttributes(out, grokExec.extractedFields());
+        out.writeNamedWriteableCollection(grokExec.extractedFields());
     }
 
     static LimitExec readLimitExec(PlanStreamInput in) throws IOException {
@@ -713,14 +721,19 @@ static void writeLimitExec(PlanStreamOutput out, LimitExec limitExec) throws IOE
     }
 
     static MvExpandExec readMvExpandExec(PlanStreamInput in) throws IOException {
-        return new MvExpandExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readNamedExpression(), in.readAttribute());
+        return new MvExpandExec(
+            Source.readFrom(in),
+            in.readPhysicalPlanNode(),
+            in.readNamedExpression(),
+            in.readNamedWriteable(Attribute.class)
+        );
     }
 
     static void writeMvExpandExec(PlanStreamOutput out, MvExpandExec mvExpandExec) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(mvExpandExec.child());
         out.writeNamedExpression(mvExpandExec.target());
-        out.writeAttribute(mvExpandExec.expanded());
+        out.writeNamedWriteable(mvExpandExec.expanded());
     }
 
     static OrderExec readOrderExec(PlanStreamInput in) throws IOException {
@@ -759,12 +772,16 @@ static void writeRowExec(PlanStreamOutput out, RowExec rowExec) throws IOExcepti
 
     @SuppressWarnings("unchecked")
     static ShowExec readShowExec(PlanStreamInput in) throws IOException {
-        return new ShowExec(Source.readFrom(in), readAttributes(in), (List<List<Object>>) in.readGenericValue());
+        return new ShowExec(
+            Source.readFrom(in),
+            in.readNamedWriteableCollectionAsList(Attribute.class),
+            (List<List<Object>>) in.readGenericValue()
+        );
     }
 
     static void writeShowExec(PlanStreamOutput out, ShowExec showExec) throws IOException {
         Source.EMPTY.writeTo(out);
-        writeAttributes(out, showExec.output());
+        out.writeNamedWriteableCollection(showExec.output());
         out.writeGenericValue(showExec.values());
     }
 
@@ -804,7 +821,13 @@ static void writeAggregate(PlanStreamOutput out, Aggregate aggregate) throws IOE
     }
 
     static Dissect readDissect(PlanStreamInput in) throws IOException {
-        return new Dissect(Source.readFrom(in), in.readLogicalPlanNode(), in.readExpression(), readDissectParser(in), readAttributes(in));
+        return new Dissect(
+            Source.readFrom(in),
+            in.readLogicalPlanNode(),
+            in.readExpression(),
+            readDissectParser(in),
+            in.readNamedWriteableCollectionAsList(Attribute.class)
+        );
     }
 
     static void writeDissect(PlanStreamOutput out, Dissect dissect) throws IOException {
@@ -812,13 +835,13 @@ static void writeDissect(PlanStreamOutput out, Dissect dissect) throws IOExcepti
         out.writeLogicalPlanNode(dissect.child());
         out.writeExpression(dissect.input());
         writeDissectParser(out, dissect.parser());
-        writeAttributes(out, dissect.extractedFields());
+        out.writeNamedWriteableCollection(dissect.extractedFields());
     }
 
     static EsRelation readEsRelation(PlanStreamInput in) throws IOException {
         Source source = Source.readFrom(in);
         EsIndex esIndex = readEsIndex(in);
-        List<Attribute> attributes = readAttributes(in);
+        List<Attribute> attributes = in.readNamedWriteableCollectionAsList(Attribute.class);
         if (supportingEsSourceOptions(in.getTransportVersion())) {
             readEsSourceOptions(in); // consume optional strings sent by remote
         }
@@ -831,7 +854,7 @@ static void writeEsRelation(PlanStreamOutput out, EsRelation relation) throws IO
         assert relation.children().size() == 0;
         Source.EMPTY.writeTo(out);
         writeEsIndex(out, relation.index());
-        writeAttributes(out, relation.output());
+        out.writeNamedWriteableCollection(relation.output());
         if (supportingEsSourceOptions(out.getTransportVersion())) {
             writeEsSourceOptions(out); // write (null) string fillers expected by remote
         }
@@ -953,7 +976,7 @@ static Grok readGrok(PlanStreamInput in) throws IOException {
             in.readLogicalPlanNode(),
             in.readExpression(),
             Grok.pattern(source, in.readString()),
-            readAttributes(in)
+            in.readNamedWriteableCollectionAsList(Attribute.class)
         );
     }
 
@@ -962,7 +985,7 @@ static void writeGrok(PlanStreamOutput out, Grok grok) throws IOException {
         out.writeLogicalPlanNode(grok.child());
         out.writeExpression(grok.input());
         out.writeString(grok.parser().pattern());
-        writeAttributes(out, grok.extractedFields());
+        out.writeNamedWriteableCollection(grok.extractedFields());
     }
 
     static Limit readLimit(PlanStreamInput in) throws IOException {
@@ -976,14 +999,19 @@ static void writeLimit(PlanStreamOutput out, Limit limit) throws IOException {
     }
 
     static MvExpand readMvExpand(PlanStreamInput in) throws IOException {
-        return new MvExpand(Source.readFrom(in), in.readLogicalPlanNode(), in.readNamedExpression(), in.readAttribute());
+        return new MvExpand(
+            Source.readFrom(in),
+            in.readLogicalPlanNode(),
+            in.readNamedExpression(),
+            in.readNamedWriteable(Attribute.class)
+        );
     }
 
     static void writeMvExpand(PlanStreamOutput out, MvExpand mvExpand) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(mvExpand.child());
         out.writeNamedExpression(mvExpand.target());
-        out.writeAttribute(mvExpand.expanded());
+        out.writeNamedWriteable(mvExpand.expanded());
     }
 
     static OrderBy readOrderBy(PlanStreamInput in) throws IOException {
@@ -1030,14 +1058,6 @@ static void writeTopN(PlanStreamOutput out, TopN topN) throws IOException {
     // -- Attributes
     //
 
-    private static List<Attribute> readAttributes(PlanStreamInput in) throws IOException {
-        return in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readAttribute));
-    }
-
-    static void writeAttributes(PlanStreamOutput out, List<Attribute> attributes) throws IOException {
-        out.writeCollection(attributes, writerFromPlanWriter(PlanStreamOutput::writeAttribute));
-    }
-
     private static List<NamedExpression> readNamedExpressions(PlanStreamInput in) throws IOException {
         return in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readNamedExpression));
     }
@@ -1054,96 +1074,6 @@ static void writeAliases(PlanStreamOutput out, List<Alias> aliases) throws IOExc
         out.writeCollection(aliases, writerFromPlanWriter(PlanNamedTypes::writeAlias));
     }
 
-    static FieldAttribute readFieldAttribute(PlanStreamInput in) throws IOException {
-        return new FieldAttribute(
-            Source.readFrom(in),
-            in.readOptionalWithReader(PlanNamedTypes::readFieldAttribute),
-            in.readString(),
-            DataType.readFrom(in),
-            in.readNamedWriteable(EsField.class),
-            in.readOptionalString(),
-            in.readEnum(Nullability.class),
-            NameId.readFrom(in),
-            in.readBoolean()
-        );
-    }
-
-    static void writeFieldAttribute(PlanStreamOutput out, FieldAttribute fieldAttribute) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeOptionalWriteable(fieldAttribute.parent() == null ? null : o -> writeFieldAttribute(out, fieldAttribute.parent()));
-        out.writeString(fieldAttribute.name());
-        out.writeString(fieldAttribute.dataType().typeName());
-        out.writeNamedWriteable(fieldAttribute.field());
-        out.writeOptionalString(fieldAttribute.qualifier());
-        out.writeEnum(fieldAttribute.nullable());
-        fieldAttribute.id().writeTo(out);
-        out.writeBoolean(fieldAttribute.synthetic());
-    }
-
-    static ReferenceAttribute readReferenceAttr(PlanStreamInput in) throws IOException {
-        return new ReferenceAttribute(
-            Source.readFrom(in),
-            in.readString(),
-            DataType.readFrom(in),
-            in.readOptionalString(),
-            in.readEnum(Nullability.class),
-            NameId.readFrom(in),
-            in.readBoolean()
-        );
-    }
-
-    static void writeReferenceAttr(PlanStreamOutput out, ReferenceAttribute referenceAttribute) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeString(referenceAttribute.name());
-        out.writeString(referenceAttribute.dataType().typeName());
-        out.writeOptionalString(referenceAttribute.qualifier());
-        out.writeEnum(referenceAttribute.nullable());
-        referenceAttribute.id().writeTo(out);
-        out.writeBoolean(referenceAttribute.synthetic());
-    }
-
-    static MetadataAttribute readMetadataAttr(PlanStreamInput in) throws IOException {
-        return new MetadataAttribute(
-            Source.readFrom(in),
-            in.readString(),
-            DataType.readFrom(in),
-            in.readOptionalString(),
-            in.readEnum(Nullability.class),
-            NameId.readFrom(in),
-            in.readBoolean(),
-            in.readBoolean()
-        );
-    }
-
-    static void writeMetadataAttr(PlanStreamOutput out, MetadataAttribute metadataAttribute) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeString(metadataAttribute.name());
-        out.writeString(metadataAttribute.dataType().typeName());
-        out.writeOptionalString(metadataAttribute.qualifier());
-        out.writeEnum(metadataAttribute.nullable());
-        metadataAttribute.id().writeTo(out);
-        out.writeBoolean(metadataAttribute.synthetic());
-        out.writeBoolean(metadataAttribute.searchable());
-    }
-
-    static UnsupportedAttribute readUnsupportedAttr(PlanStreamInput in) throws IOException {
-        return new UnsupportedAttribute(
-            Source.readFrom(in),
-            in.readString(),
-            new UnsupportedEsField(in),
-            in.readOptionalString(),
-            NameId.readFrom(in)
-        );
-    }
-
-    static void writeUnsupportedAttr(PlanStreamOutput out, UnsupportedAttribute unsupportedAttribute) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeString(unsupportedAttribute.name());
-        unsupportedAttribute.field().writeTo(out);
-        out.writeOptionalString(unsupportedAttribute.hasCustomMessage() ? unsupportedAttribute.unresolvedMessage() : null);
-        unsupportedAttribute.id().writeTo(out);
-    }
-
     // -- BinaryComparison
 
     static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) throws IOException {
@@ -1840,14 +1770,14 @@ static void writeOrder(PlanStreamOutput out, Order order) throws IOException {
 
     static EsQueryExec.FieldSort readFieldSort(PlanStreamInput in) throws IOException {
         return new EsQueryExec.FieldSort(
-            readFieldAttribute(in),
+            new FieldAttribute(in),
             in.readEnum(Order.OrderDirection.class),
             in.readEnum(Order.NullsPosition.class)
         );
     }
 
     static void writeFieldSort(PlanStreamOutput out, EsQueryExec.FieldSort fieldSort) throws IOException {
-        writeFieldAttribute(out, fieldSort.field());
+        fieldSort.field().writeTo(out);
         out.writeEnum(fieldSort.direction());
         out.writeEnum(fieldSort.nulls());
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
index 73a7ba6549dc8..e7f1fbd6e1460 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
@@ -24,8 +23,6 @@
 import org.elasticsearch.compute.data.LongBigArrayBlock;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.xpack.esql.Column;
-import org.elasticsearch.xpack.esql.core.expression.Attribute;
-import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
@@ -36,9 +33,7 @@
 import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.function.LongFunction;
 
@@ -106,10 +101,6 @@ public NamedExpression readNamedExpression() throws IOException {
         return readNamed(NamedExpression.class);
     }
 
-    public Attribute readAttribute() throws IOException {
-        return readNamed(Attribute.class);
-    }
-
     public <T> T readNamed(Class<T> type) throws IOException {
         String name = readString();
         @SuppressWarnings("unchecked")
@@ -145,18 +136,6 @@ public <T> T readOptionalWithReader(PlanReader<T> reader) throws IOException {
         }
     }
 
-    public AttributeSet readAttributeSet(Writeable.Reader<Attribute> reader) throws IOException {
-        int count = readArraySize();
-        if (count == 0) {
-            return new AttributeSet();
-        }
-        Collection<Attribute> builder = new HashSet<>();
-        for (int i = 0; i < count; i++) {
-            builder.add(reader.read(this));
-        }
-        return new AttributeSet(builder);
-    }
-
     public EsqlConfiguration configuration() throws IOException {
         return configuration;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
index 351918699aac4..05dc7ab919868 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.compute.data.LongBigArrayBlock;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.Column;
-import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
@@ -103,10 +102,6 @@ public void writeNamedExpression(NamedExpression namedExpression) throws IOExcep
         writeNamed(NamedExpression.class, namedExpression);
     }
 
-    public void writeAttribute(Attribute attribute) throws IOException {
-        writeNamed(Attribute.class, attribute);
-    }
-
     public void writeOptionalExpression(Expression expression) throws IOException {
         if (expression == null) {
             writeBoolean(false);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
index f3b06f5629524..6059b61031d1e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
@@ -55,10 +55,12 @@
 import org.elasticsearch.xpack.esql.action.RestEsqlDeleteAsyncResultAction;
 import org.elasticsearch.xpack.esql.action.RestEsqlGetAsyncResultAction;
 import org.elasticsearch.xpack.esql.action.RestEsqlQueryAction;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.index.IndexResolver;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator;
 import org.elasticsearch.xpack.esql.execution.PlanExecutor;
+import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery;
 import org.elasticsearch.xpack.esql.session.EsqlIndexResolver;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
@@ -191,6 +193,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(EnrichLookupOperator.Status.ENTRY);
         entries.addAll(Block.getNamedWriteables());
         entries.addAll(EsField.getNamedWriteables());
+        entries.addAll(Attribute.getNamedWriteables());
+        entries.add(UnsupportedAttribute.ENTRY);  // TODO combine with above once these are in the same project
         return entries;
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
index 3828b1b290c8f..6ef33b7ae5eb8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
@@ -23,9 +23,11 @@
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.query.WildcardQueryBuilder;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
@@ -114,6 +116,8 @@ public static NamedWriteableRegistry writableRegistry() {
         entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, ExistsQueryBuilder.NAME, ExistsQueryBuilder::new));
         entries.add(SingleValueQuery.ENTRY);
         entries.addAll(EsField.getNamedWriteables());
+        entries.addAll(Attribute.getNamedWriteables());
+        entries.add(UnsupportedAttribute.ENTRY);
         return new NamedWriteableRegistry(entries);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAttributeTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAttributeTestCase.java
new file mode 100644
index 0000000000000..17dcab2048eb1
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAttributeTestCase.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.session.EsqlConfigurationSerializationTests;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.hamcrest.Matchers.sameInstance;
+
+public abstract class AbstractAttributeTestCase<T extends Attribute> extends AbstractWireSerializingTestCase<
+    AbstractAttributeTestCase.ExtraAttribute> {
+    protected abstract T create();
+
+    protected abstract T mutate(T instance);
+
+    @Override
+    protected final ExtraAttribute createTestInstance() {
+        return new ExtraAttribute(create());
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected final ExtraAttribute mutateInstance(ExtraAttribute instance) {
+        return new ExtraAttribute(mutate((T) instance.a));
+    }
+
+    @Override
+    protected final NamedWriteableRegistry getNamedWriteableRegistry() {
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(Attribute.getNamedWriteables());
+        entries.add(UnsupportedAttribute.ENTRY);
+        entries.addAll(EsField.getNamedWriteables());
+        return new NamedWriteableRegistry(entries);
+    }
+
+    @Override
+    protected final Writeable.Reader<ExtraAttribute> instanceReader() {
+        return ExtraAttribute::new;
+    }
+
+    /**
+     * Adds extra equality comparisons needed for testing round trips of {@link Attribute}.
+     */
+    public static class ExtraAttribute implements Writeable {
+        private final Attribute a;
+
+        ExtraAttribute(Attribute a) {
+            this.a = a;
+            assertThat(a.source(), sameInstance(Source.EMPTY));
+        }
+
+        ExtraAttribute(StreamInput in) throws IOException {
+            PlanStreamInput ps = new PlanStreamInput(
+                in,
+                PlanNameRegistry.INSTANCE,
+                in.namedWriteableRegistry(),
+                EsqlConfigurationSerializationTests.randomConfiguration("", Map.of())
+            );
+            ps.setTransportVersion(in.getTransportVersion());
+            a = ps.readNamedWriteable(Attribute.class);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeNamedWriteable(a);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return a.equals(null);
+            }
+            if (obj.getClass() != getClass()) {
+                return a.equals(obj);
+            }
+            ExtraAttribute other = (ExtraAttribute) obj;
+            if (false == a.equals(other.a)) {
+                return false;
+            }
+            if (a instanceof FieldAttribute fa && false == fa.field().equals(((FieldAttribute) other.a).field())) {
+                return false;
+            }
+            return a.source() == Source.EMPTY;
+        }
+
+        @Override
+        public int hashCode() {
+            if (a instanceof FieldAttribute fa) {
+                return Objects.hash(a, a.source(), fa.field());
+            }
+            return Objects.hash(a, a.source());
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder b = new StringBuilder(a.toString());
+            if (a instanceof FieldAttribute fa) {
+                b.append(", field=").append(fa.field());
+            }
+            return b.toString();
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
new file mode 100644
index 0000000000000..ee542232aa30b
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function;
+
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.NameId;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.AbstractEsFieldTypeTests;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+
+public class FieldAttributeTests extends AbstractAttributeTestCase<FieldAttribute> {
+    static FieldAttribute createFieldAttribute(int maxDepth) {
+        Source source = Source.EMPTY;
+        FieldAttribute parent = maxDepth == 0 || randomBoolean() ? null : createFieldAttribute(maxDepth - 1);
+        String name = randomAlphaOfLength(5);
+        DataType type = randomFrom(DataTypes.types());
+        EsField field = AbstractEsFieldTypeTests.randomAnyEsField(maxDepth);
+        String qualifier = randomBoolean() ? null : randomAlphaOfLength(3);
+        Nullability nullability = randomFrom(Nullability.values());
+        boolean synthetic = randomBoolean();
+        return new FieldAttribute(source, parent, name, type, field, qualifier, nullability, new NameId(), synthetic);
+    }
+
+    @Override
+    protected FieldAttribute create() {
+        return createFieldAttribute(3);
+    }
+
+    @Override
+    protected FieldAttribute mutate(FieldAttribute instance) {
+        Source source = instance.source();
+        FieldAttribute parent = instance.parent();
+        String name = instance.name();
+        DataType type = instance.dataType();
+        EsField field = instance.field();
+        String qualifier = instance.qualifier();
+        Nullability nullability = instance.nullable();
+        boolean synthetic = instance.synthetic();
+        switch (between(0, 6)) {
+            case 0 -> parent = randomValueOtherThan(parent, () -> randomBoolean() ? null : createFieldAttribute(2));
+            case 1 -> name = randomAlphaOfLength(name.length() + 1);
+            case 2 -> type = randomValueOtherThan(type, () -> randomFrom(DataTypes.types()));
+            case 3 -> field = randomValueOtherThan(field, () -> AbstractEsFieldTypeTests.randomAnyEsField(3));
+            case 4 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
+            case 5 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
+            case 6 -> synthetic = false == synthetic;
+        }
+        return new FieldAttribute(source, parent, name, type, field, qualifier, nullability, new NameId(), synthetic);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java
new file mode 100644
index 0000000000000..16a83b42d10ab
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function;
+
+import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
+import org.elasticsearch.xpack.esql.core.expression.NameId;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+
+public class MetadataAttributeTests extends AbstractAttributeTestCase<MetadataAttribute> {
+    @Override
+    protected MetadataAttribute create() {
+        Source source = Source.EMPTY;
+        String name = randomAlphaOfLength(5);
+        DataType type = randomFrom(DataTypes.types());
+        String qualifier = randomBoolean() ? null : randomAlphaOfLength(3);
+        Nullability nullability = randomFrom(Nullability.values());
+        boolean synthetic = randomBoolean();
+        boolean searchable = randomBoolean();
+        return new MetadataAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic, searchable);
+    }
+
+    @Override
+    protected MetadataAttribute mutate(MetadataAttribute instance) {
+        Source source = instance.source();
+        String name = instance.name();
+        DataType type = instance.dataType();
+        String qualifier = instance.qualifier();
+        Nullability nullability = instance.nullable();
+        boolean synthetic = instance.synthetic();
+        boolean searchable = instance.searchable();
+        switch (between(0, 5)) {
+            case 0 -> name = randomAlphaOfLength(name.length() + 1);
+            case 1 -> type = randomValueOtherThan(type, () -> randomFrom(DataTypes.types()));
+            case 2 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
+            case 3 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
+            case 4 -> synthetic = false == synthetic;
+            case 5 -> searchable = false == searchable;
+        }
+        return new MetadataAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic, searchable);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
new file mode 100644
index 0000000000000..e248b741ff48d
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function;
+
+import org.elasticsearch.xpack.esql.core.expression.NameId;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+
+public class ReferenceAttributeTests extends AbstractAttributeTestCase<ReferenceAttribute> {
+    @Override
+    protected ReferenceAttribute create() {
+        Source source = Source.EMPTY;
+        String name = randomAlphaOfLength(5);
+        DataType type = randomFrom(DataTypes.types());
+        String qualifier = randomBoolean() ? null : randomAlphaOfLength(3);
+        Nullability nullability = randomFrom(Nullability.values());
+        boolean synthetic = randomBoolean();
+        return new ReferenceAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic);
+    }
+
+    @Override
+    protected ReferenceAttribute mutate(ReferenceAttribute instance) {
+        Source source = instance.source();
+        String name = instance.name();
+        DataType type = instance.dataType();
+        String qualifier = instance.qualifier();
+        Nullability nullability = instance.nullable();
+        boolean synthetic = instance.synthetic();
+        switch (between(0, 4)) {
+            case 0 -> name = randomAlphaOfLength(name.length() + 1);
+            case 1 -> type = randomValueOtherThan(type, () -> randomFrom(DataTypes.types()));
+            case 2 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
+            case 3 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
+            case 4 -> synthetic = false == synthetic;
+        }
+        return new ReferenceAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttributeTests.java
new file mode 100644
index 0000000000000..e195f31664774
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttributeTests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function;
+
+import org.elasticsearch.xpack.esql.core.expression.NameId;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
+import org.elasticsearch.xpack.esql.core.type.UnsupportedEsFieldTests;
+
+public class UnsupportedAttributeTests extends AbstractAttributeTestCase<UnsupportedAttribute> {
+    @Override
+    protected UnsupportedAttribute create() {
+        String name = randomAlphaOfLength(5);
+        UnsupportedEsField field = UnsupportedEsFieldTests.randomUnsupportedEsField(4);
+        String customMessage = randomBoolean() ? null : randomAlphaOfLength(9);
+        NameId id = new NameId();
+        return new UnsupportedAttribute(Source.EMPTY, name, field, customMessage, id);
+    }
+
+    @Override
+    protected UnsupportedAttribute mutate(UnsupportedAttribute instance) {
+        Source source = instance.source();
+        String name = instance.name();
+        UnsupportedEsField field = instance.field();
+        String customMessage = instance.hasCustomMessage() ? instance.unresolvedMessage() : null;
+        switch (between(0, 2)) {
+            case 0 -> name = randomAlphaOfLength(name.length() + 1);
+            case 1 -> field = randomValueOtherThan(field, () -> UnsupportedEsFieldTests.randomUnsupportedEsField(4));
+            case 2 -> customMessage = randomValueOtherThan(customMessage, () -> randomBoolean() ? null : randomAlphaOfLength(9));
+            default -> throw new IllegalArgumentException();
+        }
+        return new UnsupportedAttribute(source, name, field, customMessage, new NameId());
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
index c1a7f1219c5f7..9cb4b6cff3fc0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
-import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.ArithmeticOperation;
@@ -219,52 +218,6 @@ public void testWrappedStreamSimple() throws IOException {
         assertThat(in.readVInt(), equalTo(11_345));
     }
 
-    public void testUnsupportedAttributeSimple() throws IOException {
-        var orig = new UnsupportedAttribute(
-            Source.EMPTY,
-            "foo",
-            new UnsupportedEsField("foo", "keyword"),
-            "field not supported",
-            new NameId()
-        );
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeUnsupportedAttr(out, orig);
-        var in = planStreamInput(bso);
-        var deser = PlanNamedTypes.readUnsupportedAttr(in);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-        assertThat(deser.id(), equalTo(in.mapNameId(Long.parseLong(orig.id().toString()))));
-    }
-
-    public void testUnsupportedAttribute() {
-        Stream.generate(PlanNamedTypesTests::randomUnsupportedAttribute).limit(100).forEach(PlanNamedTypesTests::assertNamedExpression);
-    }
-
-    public void testFieldAttributeSimple() throws IOException {
-        var orig = new FieldAttribute(
-            Source.EMPTY,
-            null, // parent, can be null
-            "bar", // name
-            DataTypes.KEYWORD,
-            randomEsField(),
-            null, // qualifier, can be null
-            Nullability.TRUE,
-            new NameId(),
-            true // synthetic
-        );
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeFieldAttribute(out, orig);
-        var in = planStreamInput(bso);
-        var deser = PlanNamedTypes.readFieldAttribute(in);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-        assertThat(deser.id(), equalTo(in.mapNameId(Long.parseLong(orig.id().toString()))));
-    }
-
-    public void testFieldAttribute() {
-        Stream.generate(PlanNamedTypesTests::randomFieldAttribute).limit(100).forEach(PlanNamedTypesTests::assertNamedExpression);
-    }
-
     public void testBinComparisonSimple() throws IOException {
         var orig = new Equals(Source.EMPTY, field("foo", DataTypes.DOUBLE), field("bar", DataTypes.DOUBLE));
         BytesStreamOutput bso = new BytesStreamOutput();
@@ -445,11 +398,6 @@ public void testMvExpand() throws IOException {
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
     }
 
-    private static void assertNamedExpression(NamedExpression origObj) {
-        var deserObj = serializeDeserialize(origObj, PlanStreamOutput::writeExpression, PlanStreamInput::readNamedExpression);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(origObj, unused -> deserObj);
-    }
-
     private static <T> void assertNamedType(Class<T> type, T origObj) {
         var deserObj = serializeDeserialize(origObj, (o, v) -> o.writeNamed(type, origObj), i -> i.readNamed(type));
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(origObj, unused -> deserObj);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
index 2b2a4e76c5cc8..dde39b66664de 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
@@ -58,8 +57,7 @@ protected Writeable.Reader<DataNodeRequest> instanceReader() {
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         List<NamedWriteableRegistry.Entry> writeables = new ArrayList<>();
         writeables.addAll(new SearchModule(Settings.EMPTY, List.of()).getNamedWriteables());
-        writeables.addAll(Block.getNamedWriteables());
-        writeables.addAll(EsField.getNamedWriteables());
+        writeables.addAll(new EsqlPlugin().getNamedWriteables());
         return new NamedWriteableRegistry(writeables);
     }
 

From 1dc6639dfa132d751d19f8d8ab443192bb74f1e1 Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Mon, 3 Jun 2024 11:07:12 -0400
Subject: [PATCH 112/208] migrate TyperResolutionTests and fix imports
 (#109297)

The TyperResolutionTests were pointing to the core version of Mul; This PR changes them to refer to the code we actually run.
---
 .../xpack/esql}/expression/TyperResolutionTests.java         | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
 rename x-pack/plugin/{esql-core/src/test/java/org/elasticsearch/xpack/esql/core => esql/src/test/java/org/elasticsearch/xpack/esql}/expression/TyperResolutionTests.java (80%)

diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/TyperResolutionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/TyperResolutionTests.java
similarity index 80%
rename from x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/TyperResolutionTests.java
rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/TyperResolutionTests.java
index 213c29040a4b2..c2031cb0a8efa 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/TyperResolutionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/TyperResolutionTests.java
@@ -5,12 +5,13 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.core.expression;
+package org.elasticsearch.xpack.esql.expression;
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.TestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression.TypeResolution;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.Mul;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul;
 
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 

From 4b9f851400ea0c32a8b6f02209fc9a79599739c4 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Mon, 3 Jun 2024 17:11:11 +0200
Subject: [PATCH 113/208] [Inference API] Remove
 HuggingFaceElserSecretSettings.java and use DefaultSecretSettings instead
 (#109295)

---
 .../InferenceNamedWriteablesProvider.java     |  4 -
 .../elser/HuggingFaceElserModel.java          |  9 +-
 .../elser/HuggingFaceElserSecretSettings.java | 78 ------------------
 .../elser/HuggingFaceElserModelTests.java     |  5 +-
 .../HuggingFaceElserSecretSettingsTests.java  | 82 -------------------
 5 files changed, 8 insertions(+), 170 deletions(-)
 delete mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettings.java
 delete mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettingsTests.java

diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
index 506cef4905875..bff7ecdcc4a07 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java
@@ -47,7 +47,6 @@
 import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionServiceSettings;
 import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsServiceSettings;
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceServiceSettings;
-import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserSecretSettings;
 import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserServiceSettings;
 import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettings;
 import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionServiceSettings;
@@ -266,9 +265,6 @@ private static void addHuggingFaceNamedWriteables(List<NamedWriteableRegistry.En
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(ServiceSettings.class, HuggingFaceServiceSettings.NAME, HuggingFaceServiceSettings::new)
         );
-        namedWriteables.add(
-            new NamedWriteableRegistry.Entry(SecretSettings.class, HuggingFaceElserSecretSettings.NAME, HuggingFaceElserSecretSettings::new)
-        );
     }
 
     private static void addGoogleAiStudioNamedWritables(List<NamedWriteableRegistry.Entry> namedWriteables) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java
index 8a947ce9a024b..9010571ea2e55 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.huggingface.HuggingFaceActionVisitor;
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceModel;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 import java.util.Map;
 
@@ -30,7 +31,7 @@ public HuggingFaceElserModel(
             taskType,
             service,
             HuggingFaceElserServiceSettings.fromMap(serviceSettings),
-            HuggingFaceElserSecretSettings.fromMap(secrets)
+            DefaultSecretSettings.fromMap(secrets)
         );
     }
 
@@ -39,7 +40,7 @@ public HuggingFaceElserModel(
         TaskType taskType,
         String service,
         HuggingFaceElserServiceSettings serviceSettings,
-        @Nullable HuggingFaceElserSecretSettings secretSettings
+        @Nullable DefaultSecretSettings secretSettings
     ) {
         super(
             new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings),
@@ -55,8 +56,8 @@ public HuggingFaceElserServiceSettings getServiceSettings() {
     }
 
     @Override
-    public HuggingFaceElserSecretSettings getSecretSettings() {
-        return (HuggingFaceElserSecretSettings) super.getSecretSettings();
+    public DefaultSecretSettings getSecretSettings() {
+        return (DefaultSecretSettings) super.getSecretSettings();
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettings.java
deleted file mode 100644
index 48c8997f2a1bd..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettings.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.services.huggingface.elser;
-
-import org.elasticsearch.TransportVersion;
-import org.elasticsearch.TransportVersions;
-import org.elasticsearch.common.ValidationException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SecretSettings;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.inference.services.settings.ApiKeySecrets;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredSecureString;
-
-public record HuggingFaceElserSecretSettings(SecureString apiKey) implements SecretSettings, ApiKeySecrets {
-    public static final String NAME = "hugging_face_elser_secret_settings";
-
-    static final String API_KEY = "api_key";
-
-    public static HuggingFaceElserSecretSettings fromMap(@Nullable Map<String, Object> map) {
-        if (map == null) {
-            return null;
-        }
-
-        ValidationException validationException = new ValidationException();
-        SecureString secureApiToken = extractRequiredSecureString(map, API_KEY, ModelSecrets.SECRET_SETTINGS, validationException);
-
-        if (validationException.validationErrors().isEmpty() == false) {
-            throw validationException;
-        }
-
-        return new HuggingFaceElserSecretSettings(secureApiToken);
-    }
-
-    public HuggingFaceElserSecretSettings {
-        Objects.requireNonNull(apiKey);
-    }
-
-    public HuggingFaceElserSecretSettings(StreamInput in) throws IOException {
-        this(in.readSecureString());
-    }
-
-    @Override
-    public String getWriteableName() {
-        return NAME;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.field(API_KEY, apiKey.toString());
-        builder.endObject();
-        return builder;
-    }
-
-    @Override
-    public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.V_8_12_0;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeSecureString(apiKey);
-    }
-}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModelTests.java
index 2ad2c12b4a97c..d7a62256f8d9c 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModelTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModelTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 import static org.hamcrest.Matchers.containsString;
 
@@ -26,7 +27,7 @@ public static HuggingFaceElserModel createModel(String url, String apiKey) {
             TaskType.SPARSE_EMBEDDING,
             "service",
             new HuggingFaceElserServiceSettings(url),
-            new HuggingFaceElserSecretSettings(new SecureString(apiKey.toCharArray()))
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
         );
     }
 
@@ -36,7 +37,7 @@ public static HuggingFaceElserModel createModel(String url, String apiKey, Strin
             TaskType.SPARSE_EMBEDDING,
             "service",
             new HuggingFaceElserServiceSettings(url),
-            new HuggingFaceElserSecretSettings(new SecureString(apiKey.toCharArray()))
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
         );
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettingsTests.java
deleted file mode 100644
index f69a9b5a967e0..0000000000000
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettingsTests.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.services.huggingface.elser;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.ValidationException;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.is;
-
-public class HuggingFaceElserSecretSettingsTests extends AbstractWireSerializingTestCase<HuggingFaceElserSecretSettings> {
-
-    public static HuggingFaceElserSecretSettings createRandom() {
-        return new HuggingFaceElserSecretSettings(new SecureString(randomAlphaOfLength(15).toCharArray()));
-    }
-
-    public void testFromMap() {
-        var apiKey = "abc";
-        var serviceSettings = HuggingFaceElserSecretSettings.fromMap(new HashMap<>(Map.of(HuggingFaceElserSecretSettings.API_KEY, apiKey)));
-
-        assertThat(new HuggingFaceElserSecretSettings(new SecureString(apiKey.toCharArray())), is(serviceSettings));
-    }
-
-    public void testFromMap_ReturnsNull_WhenMapIsNull() {
-        assertNull(HuggingFaceElserSecretSettings.fromMap(null));
-    }
-
-    public void testFromMap_MissingApiKey_ThrowsError() {
-        var thrownException = expectThrows(ValidationException.class, () -> HuggingFaceElserSecretSettings.fromMap(new HashMap<>()));
-
-        assertThat(
-            thrownException.getMessage(),
-            containsString(
-                Strings.format("[secret_settings] does not contain the required setting [%s]", HuggingFaceElserSecretSettings.API_KEY)
-            )
-        );
-    }
-
-    public void testFromMap_EmptyApiKey_ThrowsError() {
-        var thrownException = expectThrows(
-            ValidationException.class,
-            () -> HuggingFaceElserSecretSettings.fromMap(new HashMap<>(Map.of(HuggingFaceElserSecretSettings.API_KEY, "")))
-        );
-
-        assertThat(
-            thrownException.getMessage(),
-            containsString(
-                Strings.format(
-                    "[secret_settings] Invalid value empty string. [%s] must be a non-empty string",
-                    HuggingFaceElserSecretSettings.API_KEY
-                )
-            )
-        );
-    }
-
-    @Override
-    protected Writeable.Reader<HuggingFaceElserSecretSettings> instanceReader() {
-        return HuggingFaceElserSecretSettings::new;
-    }
-
-    @Override
-    protected HuggingFaceElserSecretSettings createTestInstance() {
-        return createRandom();
-    }
-
-    @Override
-    protected HuggingFaceElserSecretSettings mutateInstance(HuggingFaceElserSecretSettings instance) throws IOException {
-        return randomValueOtherThan(instance, HuggingFaceElserSecretSettingsTests::createRandom);
-    }
-}

From d12a31830408e3c33da50e23cf8a82ecc81ff292 Mon Sep 17 00:00:00 2001
From: shainaraskas <58563081+shainaraskas@users.noreply.github.com>
Date: Mon, 3 Jun 2024 11:29:00 -0400
Subject: [PATCH 114/208] change  from string to array in example (#109263)

---
 docs/reference/docs/reindex.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 146b519b05e80..dc27e40ecd90b 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -1035,7 +1035,7 @@ ignored, only the host and port are used. For example:
 
 [source,yaml]
 --------------------------------------------------
-reindex.remote.whitelist: "otherhost:9200, another:9200, 127.0.10.*:9200, localhost:*"
+reindex.remote.whitelist: [otherhost:9200, another:9200, 127.0.10.*:9200, localhost:*"]
 --------------------------------------------------
 
 The list of allowed hosts must be configured on any nodes that will coordinate the reindex.

From d1da412a3e91b5736057c2783062831c52490815 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?=
 <istvan.szabo@elastic.co>
Date: Mon, 3 Jun 2024 17:32:31 +0200
Subject: [PATCH 115/208] [DOCS] Expands DELETE inference API docs (#109282)

* [DOCS] Expands DELETE inference API docs.

* [DOCS] Adds discrete flag.
---
 .../reference/inference/delete-inference.asciidoc | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/docs/reference/inference/delete-inference.asciidoc b/docs/reference/inference/delete-inference.asciidoc
index 89f76e6cef841..dca800c98ca2e 100644
--- a/docs/reference/inference/delete-inference.asciidoc
+++ b/docs/reference/inference/delete-inference.asciidoc
@@ -43,6 +43,21 @@ The unique identifier of the {infer} endpoint to delete.
 The type of {infer} task that the model performs.
 
 
+[discrete]
+[[delete-inference-query-parms]]
+== {api-query-parms-title}
+
+`dry_run`::
+(Optional, Boolean)
+When `true`, checks the {infer} processors that reference the endpoint and
+returns them in a list, but does not deletes the endpoint. Defaults to `false`.
+
+`force`::
+(Optional, Boolean)
+Deletes the endpoint regardless if it's used in an {infer} pipeline or a in a
+`semantic_text` field.
+
+
 [discrete]
 [[delete-inference-api-example]]
 ==== {api-examples-title}

From f5957510693be23d2939849608e66fbb16004647 Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Mon, 3 Jun 2024 10:43:35 -0500
Subject: [PATCH 116/208] Test to ensure that remote CCS and remote CCR actions
 don't overlap (#108590)

This commit introduces a test to help ensure that actions used between CCR and CCS
are evaluated on case-by-case basis if/when the allowed permissions overlap.
Some cases are fine if they overlap, other cases it could be a security bug.
This will test will fail the permissions we assign to the RCS 2.0 API key overlap.
---
 x-pack/plugin/core/build.gradle               | 13 +++
 .../core/security/support/Automatons.java     | 34 +++++++-
 .../authz/privilege/IndexPrivilegeTests.java  |  1 +
 .../support/AutomatonPatternsTests.java       | 81 +++++++++++++++++++
 4 files changed, 127 insertions(+), 2 deletions(-)
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonPatternsTests.java

diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index fb35b34fd4dfd..f0a00b7aa7e75 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -180,3 +180,16 @@ if (BuildParams.inFipsJvm) {
   // Test clusters run with security disabled
   tasks.named("javaRestTest").configure { enabled = false }
 }
+
+//this specific test requires a test only system property to be set, so we run it in a different JVM via a separate task
+tasks.register('testAutomatonPatterns', Test) {
+  include '**/AutomatonPatternsTests.class'
+  systemProperty 'tests.automaton.record.patterns', 'true'
+  testClassesDirs = sourceSets.test.output.classesDirs
+  classpath = sourceSets.test.runtimeClasspath
+}
+
+tasks.named('test').configure {
+  exclude '**/AutomatonPatternsTests.class'
+  dependsOn testAutomatonPatterns //to ensure testAutomatonPatterns are run with the test task
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
index f601aa144aa00..a6347d8b7ec77 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java
@@ -23,12 +23,16 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Function;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import static org.apache.lucene.util.automaton.Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
 import static org.apache.lucene.util.automaton.Operations.concatenate;
@@ -69,6 +73,10 @@ public final class Automatons {
     static final char WILDCARD_CHAR = '?';       // Char equality with support for wildcards
     static final char WILDCARD_ESCAPE = '\\';    // Escape character
 
+    // for testing only -Dtests.jvm.argline="-Dtests.automaton.record.patterns=true"
+    public static boolean recordPatterns = System.getProperty("tests.automaton.record.patterns", "false").equals("true");
+    private static final Map<Automaton, List<String>> patternsMap = new HashMap<>();
+
     private Automatons() {}
 
     /**
@@ -87,10 +95,13 @@ public static Automaton patterns(Collection<String> patterns) {
             return EMPTY;
         }
         if (cache == null) {
-            return buildAutomaton(patterns);
+            return maybeRecordPatterns(buildAutomaton(patterns), patterns);
         } else {
             try {
-                return cache.computeIfAbsent(Sets.newHashSet(patterns), p -> buildAutomaton((Set<String>) p));
+                return cache.computeIfAbsent(
+                    Sets.newHashSet(patterns),
+                    p -> maybeRecordPatterns(buildAutomaton((Set<String>) p), patterns)
+                );
             } catch (ExecutionException e) {
                 throw unwrapCacheException(e);
             }
@@ -338,4 +349,23 @@ public static void addSettings(List<Setting<?>> settingsList) {
         settingsList.add(CACHE_SIZE);
         settingsList.add(CACHE_TTL);
     }
+
+    private static Automaton maybeRecordPatterns(Automaton automaton, Collection<String> patterns) {
+        if (recordPatterns) {
+            patternsMap.put(
+                automaton,
+                patterns.stream().map(String::trim).map(s -> s.toLowerCase(Locale.ROOT)).sorted().collect(Collectors.toList())
+            );
+        }
+        return automaton;
+    }
+
+    // test only
+    static List<String> getPatterns(Automaton automaton) {
+        if (recordPatterns) {
+            return patternsMap.get(automaton);
+        } else {
+            throw new IllegalArgumentException("recordPatterns is set to false");
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java
index b05f7065ff63c..265714ee6ea16 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java
@@ -144,4 +144,5 @@ public void testCrossClusterReplicationPrivileges() {
         );
         assertThat(Operations.subsetOf(crossClusterReplicationInternal.automaton, IndexPrivilege.get(Set.of("all")).automaton), is(true));
     }
+
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonPatternsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonPatternsTests.java
new file mode 100644
index 0000000000000..1539651b1aed6
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonPatternsTests.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.support;
+
+import org.apache.lucene.util.automaton.Automaton;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.core.security.action.apikey.CrossClusterApiKeyRoleDescriptorBuilder.CCR_INDICES_PRIVILEGE_NAMES;
+import static org.elasticsearch.xpack.core.security.action.apikey.CrossClusterApiKeyRoleDescriptorBuilder.CCS_INDICES_PRIVILEGE_NAMES;
+
+public class AutomatonPatternsTests extends ESTestCase {
+
+    /**
+     * RCS 2.0 allows a single API key to define "replication" and "search" blocks. If both are defined, this results in an API key with 2
+     * sets of indices permissions. Due to the way API keys (and roles) work across the multiple index permission, the set of index
+     * patterns allowed are effectively the most generous of the sets of index patterns since the index patterns are OR'ed together. For
+     * example, `foo` OR `*` results in access to `*`. So, if you have "search" access defined as `foo`, but replication access defined
+     * as `*`, the API key effectively allows access to index pattern `*`. This means that the access for API keys that define both
+     * "search" and "replication", the action names used are the primary means by which we can constrain CCS to the set of "search" indices
+     * as well as how we constrain CCR to the set "replication" indices. For example, if "replication" ever allowed access to
+     * `indices:data/read/get` for `*` , then the "replication" permissions would effectively enable users of CCS to get documents,
+     * even if "search" is never defined in the RCS 2.0 API key. This obviously is not desirable and in practice when both "search" and
+     * "replication" are defined the isolation between CCS and CCR is only achieved because the action names for the workflows do not
+     * overlap. This test helps to ensure that the actions names used for RCS 2.0 do not bleed over between search and replication.
+     */
+    public void testRemoteClusterPrivsDoNotOverlap() {
+
+        // check that the action patterns for remote CCS are not allowed by remote CCR privileges
+        Arrays.stream(CCS_INDICES_PRIVILEGE_NAMES).forEach(ccsPrivilege -> {
+            Automaton ccsAutomaton = IndexPrivilege.get(Set.of(ccsPrivilege)).getAutomaton();
+            Automatons.getPatterns(ccsAutomaton).forEach(ccsPattern -> {
+                // emulate an action name that could be allowed by a CCS privilege
+                String actionName = ccsPattern.replaceAll("\\*", randomAlphaOfLengthBetween(1, 8));
+                Arrays.stream(CCR_INDICES_PRIVILEGE_NAMES).forEach(ccrPrivileges -> {
+                    String errorMessage = String.format(
+                        Locale.ROOT,
+                        "CCR privilege \"%s\" allows CCS action \"%s\". This could result in an "
+                            + "accidental bleeding of permission between RCS 2.0's search and replication index permissions",
+                        ccrPrivileges,
+                        ccsPattern
+                    );
+                    assertFalse(errorMessage, IndexPrivilege.get(Set.of(ccrPrivileges)).predicate().test(actionName));
+                });
+            });
+        });
+
+        // check that the action patterns for remote CCR are not allowed by remote CCS privileges
+        Arrays.stream(CCR_INDICES_PRIVILEGE_NAMES).forEach(ccrPrivilege -> {
+            Automaton ccrAutomaton = IndexPrivilege.get(Set.of(ccrPrivilege)).getAutomaton();
+            Automatons.getPatterns(ccrAutomaton).forEach(ccrPattern -> {
+                // emulate an action name that could be allowed by a CCR privilege
+                String actionName = ccrPattern.replaceAll("\\*", randomAlphaOfLengthBetween(1, 8));
+                Arrays.stream(CCS_INDICES_PRIVILEGE_NAMES).forEach(ccsPrivileges -> {
+                    if ("indices:data/read/xpack/ccr/shard_changes*".equals(ccrPattern)) {
+                        // do nothing, this action is only applicable to CCR workflows and is a moot point if CCS technically has
+                        // access to the index pattern for this action granted by CCR
+                    } else {
+                        String errorMessage = String.format(
+                            Locale.ROOT,
+                            "CCS privilege \"%s\" allows CCR action \"%s\". This could result in an accidental bleeding of "
+                                + "permission between RCS 2.0's search and replication index permissions",
+                            ccsPrivileges,
+                            ccrPattern
+                        );
+                        assertFalse(errorMessage, IndexPrivilege.get(Set.of(ccsPrivileges)).predicate().test(actionName));
+                    }
+                });
+            });
+        });
+    }
+}

From ac0fa4d4d6c02af05f5ec694a8ce107c7967dec1 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Mon, 3 Jun 2024 18:10:46 +0200
Subject: [PATCH 117/208] [Inference API] Remove GoogleAiStudioSecretSettings
 and use DefaultSecretSettings (#109296)

---
 .../googleaistudio/GoogleAiStudioRequest.java |   4 +-
 .../GoogleAiStudioSecretSettings.java         | 106 ------------------
 .../GoogleAiStudioCompletionModel.java        |  12 +-
 .../GoogleAiStudioEmbeddingsModel.java        |  12 +-
 .../GoogleAiStudioRequestTests.java           |   6 +-
 .../GoogleAiStudioSecretSettingsTests.java    |  71 ------------
 .../GoogleAiStudioCompletionModelTests.java   |   6 +-
 .../GoogleAiStudioEmbeddingsModelTests.java   |   8 +-
 8 files changed, 24 insertions(+), 201 deletions(-)
 delete mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioSecretSettings.java
 delete mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioSecretSettingsTests.java

diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java
index ede9c6193aa21..fb99deabc9c5e 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequest.java
@@ -11,13 +11,13 @@
 import org.apache.http.client.utils.URIBuilder;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.xpack.inference.external.request.Request;
-import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 public interface GoogleAiStudioRequest extends Request {
 
     String API_KEY_PARAMETER = "key";
 
-    static void decorateWithApiKeyParameter(HttpPost httpPost, GoogleAiStudioSecretSettings secretSettings) {
+    static void decorateWithApiKeyParameter(HttpPost httpPost, DefaultSecretSettings secretSettings) {
         try {
             var uri = httpPost.getURI();
             var uriWithApiKey = new URIBuilder().setScheme(uri.getScheme())
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioSecretSettings.java
deleted file mode 100644
index bf702d010e2a8..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioSecretSettings.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.services.googleaistudio;
-
-import org.elasticsearch.TransportVersion;
-import org.elasticsearch.TransportVersions;
-import org.elasticsearch.common.ValidationException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.inference.ModelSecrets;
-import org.elasticsearch.inference.SecretSettings;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.core.Strings.format;
-import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalSecureString;
-
-public class GoogleAiStudioSecretSettings implements SecretSettings {
-
-    public static final String NAME = "google_ai_studio_secret_settings";
-    public static final String API_KEY = "api_key";
-
-    private final SecureString apiKey;
-
-    public static GoogleAiStudioSecretSettings fromMap(@Nullable Map<String, Object> map) {
-        if (map == null) {
-            return null;
-        }
-
-        ValidationException validationException = new ValidationException();
-        SecureString secureApiKey = extractOptionalSecureString(map, API_KEY, ModelSecrets.SECRET_SETTINGS, validationException);
-
-        if (secureApiKey == null) {
-            validationException.addValidationError(format("[secret_settings] must have [%s] set", API_KEY));
-        }
-
-        if (validationException.validationErrors().isEmpty() == false) {
-            throw validationException;
-        }
-
-        return new GoogleAiStudioSecretSettings(secureApiKey);
-    }
-
-    public GoogleAiStudioSecretSettings(SecureString apiKey) {
-        Objects.requireNonNull(apiKey);
-        this.apiKey = apiKey;
-    }
-
-    public GoogleAiStudioSecretSettings(StreamInput in) throws IOException {
-        this(in.readOptionalSecureString());
-    }
-
-    public SecureString apiKey() {
-        return apiKey;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-
-        if (apiKey != null) {
-            builder.field(API_KEY, apiKey.toString());
-        }
-
-        builder.endObject();
-        return builder;
-    }
-
-    @Override
-    public String getWriteableName() {
-        return NAME;
-    }
-
-    @Override
-    public TransportVersion getMinimalSupportedVersion() {
-        return TransportVersions.ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeOptionalSecureString(apiKey);
-    }
-
-    @Override
-    public boolean equals(Object object) {
-        if (this == object) return true;
-        if (object == null || getClass() != object.getClass()) return false;
-        GoogleAiStudioSecretSettings that = (GoogleAiStudioSecretSettings) object;
-        return Objects.equals(apiKey, that.apiKey);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(apiKey);
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java
index 6a11f678158b6..eafb0c372202c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioUtils;
 import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioModel;
-import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -45,7 +45,7 @@ public GoogleAiStudioCompletionModel(
             service,
             GoogleAiStudioCompletionServiceSettings.fromMap(serviceSettings),
             EmptyTaskSettings.INSTANCE,
-            GoogleAiStudioSecretSettings.fromMap(secrets)
+            DefaultSecretSettings.fromMap(secrets)
         );
     }
 
@@ -56,7 +56,7 @@ public GoogleAiStudioCompletionModel(
         String service,
         GoogleAiStudioCompletionServiceSettings serviceSettings,
         TaskSettings taskSettings,
-        @Nullable GoogleAiStudioSecretSettings secrets
+        @Nullable DefaultSecretSettings secrets
     ) {
         super(
             new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings),
@@ -78,7 +78,7 @@ public GoogleAiStudioCompletionModel(
         String url,
         GoogleAiStudioCompletionServiceSettings serviceSettings,
         TaskSettings taskSettings,
-        @Nullable GoogleAiStudioSecretSettings secrets
+        @Nullable DefaultSecretSettings secrets
     ) {
         super(
             new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings),
@@ -102,8 +102,8 @@ public GoogleAiStudioCompletionServiceSettings getServiceSettings() {
     }
 
     @Override
-    public GoogleAiStudioSecretSettings getSecretSettings() {
-        return (GoogleAiStudioSecretSettings) super.getSecretSettings();
+    public DefaultSecretSettings getSecretSettings() {
+        return (DefaultSecretSettings) super.getSecretSettings();
     }
 
     public static URI buildUri(String model) throws URISyntaxException {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
index 0794e47d338bc..ad106797de51b 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioUtils;
 import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioModel;
-import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -45,7 +45,7 @@ public GoogleAiStudioEmbeddingsModel(
             service,
             GoogleAiStudioEmbeddingsServiceSettings.fromMap(serviceSettings),
             EmptyTaskSettings.INSTANCE,
-            GoogleAiStudioSecretSettings.fromMap(secrets)
+            DefaultSecretSettings.fromMap(secrets)
         );
     }
 
@@ -60,7 +60,7 @@ public GoogleAiStudioEmbeddingsModel(GoogleAiStudioEmbeddingsModel model, Google
         String service,
         GoogleAiStudioEmbeddingsServiceSettings serviceSettings,
         TaskSettings taskSettings,
-        @Nullable GoogleAiStudioSecretSettings secrets
+        @Nullable DefaultSecretSettings secrets
     ) {
         super(
             new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings),
@@ -82,7 +82,7 @@ public GoogleAiStudioEmbeddingsModel(GoogleAiStudioEmbeddingsModel model, Google
         String uri,
         GoogleAiStudioEmbeddingsServiceSettings serviceSettings,
         TaskSettings taskSettings,
-        @Nullable GoogleAiStudioSecretSettings secrets
+        @Nullable DefaultSecretSettings secrets
     ) {
         super(
             new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings),
@@ -102,8 +102,8 @@ public GoogleAiStudioEmbeddingsServiceSettings getServiceSettings() {
     }
 
     @Override
-    public GoogleAiStudioSecretSettings getSecretSettings() {
-        return (GoogleAiStudioSecretSettings) super.getSecretSettings();
+    public DefaultSecretSettings getSecretSettings() {
+        return (DefaultSecretSettings) super.getSecretSettings();
     }
 
     public URI uri() {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequestTests.java
index d77c88dacd06f..da6070f1f455f 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequestTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/googleaistudio/GoogleAiStudioRequestTests.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -28,7 +28,7 @@ public void testDecorateWithApiKeyParameter() throws URISyntaxException {
         var uriString = "https://localhost:3000";
         var secureApiKey = new SecureString("api_key".toCharArray());
         var httpPost = new HttpPost(uriString);
-        var secretSettings = new GoogleAiStudioSecretSettings(secureApiKey);
+        var secretSettings = new DefaultSecretSettings(secureApiKey);
 
         GoogleAiStudioRequest.decorateWithApiKeyParameter(httpPost, secretSettings);
 
@@ -45,7 +45,7 @@ public void testDecorateWithApiKeyParameter_ThrowsValidationException_WhenAnyExc
             ValidationException.class,
             () -> GoogleAiStudioRequest.decorateWithApiKeyParameter(
                 httpPost,
-                new GoogleAiStudioSecretSettings(new SecureString("abc".toCharArray()))
+                new DefaultSecretSettings(new SecureString("abc".toCharArray()))
             )
         );
         assertThat(validationException.getCause(), is(cause));
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioSecretSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioSecretSettingsTests.java
deleted file mode 100644
index a0339934783d8..0000000000000
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioSecretSettingsTests.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.services.googleaistudio;
-
-import org.elasticsearch.common.ValidationException;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.is;
-
-public class GoogleAiStudioSecretSettingsTests extends AbstractWireSerializingTestCase<GoogleAiStudioSecretSettings> {
-
-    public static GoogleAiStudioSecretSettings createRandom() {
-        return new GoogleAiStudioSecretSettings(randomSecureStringOfLength(15));
-    }
-
-    public void testFromMap() {
-        var apiKey = "abc";
-        var secretSettings = GoogleAiStudioSecretSettings.fromMap(new HashMap<>(Map.of(GoogleAiStudioSecretSettings.API_KEY, apiKey)));
-
-        assertThat(new GoogleAiStudioSecretSettings(new SecureString(apiKey.toCharArray())), is(secretSettings));
-    }
-
-    public void testFromMap_ReturnsNull_WhenMapIsNull() {
-        assertNull(GoogleAiStudioSecretSettings.fromMap(null));
-    }
-
-    public void testFromMap_ThrowsError_WhenApiKeyIsNull() {
-        var throwException = expectThrows(ValidationException.class, () -> GoogleAiStudioSecretSettings.fromMap(new HashMap<>()));
-
-        assertThat(throwException.getMessage(), containsString("[secret_settings] must have [api_key] set"));
-    }
-
-    public void testFromMap_ThrowsError_WhenApiKeyIsEmpty() {
-        var thrownException = expectThrows(
-            ValidationException.class,
-            () -> GoogleAiStudioSecretSettings.fromMap(new HashMap<>(Map.of(GoogleAiStudioSecretSettings.API_KEY, "")))
-        );
-
-        assertThat(
-            thrownException.getMessage(),
-            containsString("[secret_settings] Invalid value empty string. [api_key] must be a non-empty string")
-        );
-    }
-
-    @Override
-    protected Writeable.Reader<GoogleAiStudioSecretSettings> instanceReader() {
-        return GoogleAiStudioSecretSettings::new;
-    }
-
-    @Override
-    protected GoogleAiStudioSecretSettings createTestInstance() {
-        return createRandom();
-    }
-
-    @Override
-    protected GoogleAiStudioSecretSettings mutateInstance(GoogleAiStudioSecretSettings instance) throws IOException {
-        return randomValueOtherThan(instance, GoogleAiStudioSecretSettingsTests::createRandom);
-    }
-}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java
index 1f8233f7eb103..025317fbe025a 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.inference.EmptyTaskSettings;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 import java.net.URISyntaxException;
 import java.util.HashMap;
@@ -48,7 +48,7 @@ public static GoogleAiStudioCompletionModel createModel(String model, String api
             "service",
             new GoogleAiStudioCompletionServiceSettings(model, null),
             EmptyTaskSettings.INSTANCE,
-            new GoogleAiStudioSecretSettings(new SecureString(apiKey.toCharArray()))
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
         );
     }
 
@@ -60,7 +60,7 @@ public static GoogleAiStudioCompletionModel createModel(String model, String url
             url,
             new GoogleAiStudioCompletionServiceSettings(model, null),
             EmptyTaskSettings.INSTANCE,
-            new GoogleAiStudioSecretSettings(new SecureString(apiKey.toCharArray()))
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
         );
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModelTests.java
index 770b6f8fb4137..5ea9bbfc9d970 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModelTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModelTests.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.inference.SimilarityMeasure;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioSecretSettings;
+import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 public class GoogleAiStudioEmbeddingsModelTests extends ESTestCase {
 
@@ -25,7 +25,7 @@ public static GoogleAiStudioEmbeddingsModel createModel(String model, String api
             url,
             new GoogleAiStudioEmbeddingsServiceSettings(model, null, null, SimilarityMeasure.DOT_PRODUCT, null),
             EmptyTaskSettings.INSTANCE,
-            new GoogleAiStudioSecretSettings(new SecureString(apiKey.toCharArray()))
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
         );
     }
 
@@ -43,7 +43,7 @@ public static GoogleAiStudioEmbeddingsModel createModel(
             url,
             new GoogleAiStudioEmbeddingsServiceSettings(model, null, dimensions, similarityMeasure, null),
             EmptyTaskSettings.INSTANCE,
-            new GoogleAiStudioSecretSettings(new SecureString(apiKey.toCharArray()))
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
         );
     }
 
@@ -59,7 +59,7 @@ public static GoogleAiStudioEmbeddingsModel createModel(
             "service",
             new GoogleAiStudioEmbeddingsServiceSettings(model, tokenLimit, dimensions, SimilarityMeasure.DOT_PRODUCT, null),
             EmptyTaskSettings.INSTANCE,
-            new GoogleAiStudioSecretSettings(new SecureString(apiKey.toCharArray()))
+            new DefaultSecretSettings(new SecureString(apiKey.toCharArray()))
         );
     }
 

From 2268e383e88eb103b7872397f524890eba3b669c Mon Sep 17 00:00:00 2001
From: Liam Thompson <32779855+leemthompo@users.noreply.github.com>
Date: Mon, 3 Jun 2024 18:44:33 +0200
Subject: [PATCH 118/208] [DOCS][ESQL][8.14] Add API key based security model
 info for ESQL CCS (#109155)

Co-authored-by: Jake Landis <jake.landis@elastic.co>
---
 .../esql/esql-across-clusters.asciidoc        | 126 ++++++++++++++++--
 .../cluster/remote-clusters-api-key.asciidoc  |   2 +
 2 files changed, 114 insertions(+), 14 deletions(-)

diff --git a/docs/reference/esql/esql-across-clusters.asciidoc b/docs/reference/esql/esql-across-clusters.asciidoc
index 95278314b0253..6231b4f4f0a69 100644
--- a/docs/reference/esql/esql-across-clusters.asciidoc
+++ b/docs/reference/esql/esql-across-clusters.asciidoc
@@ -1,6 +1,5 @@
 [[esql-cross-clusters]]
 === Using {esql} across clusters
-
 ++++
 <titleabbrev>Using {esql} across clusters</titleabbrev>
 ++++
@@ -11,6 +10,8 @@ preview::["{ccs-cap} for {esql} is in technical preview and may be changed or re
 
 With {esql}, you can execute a single query across multiple clusters.
 
+[discrete]
+[[esql-ccs-prerequisites]]
 ==== Prerequisites
 
 include::{es-ref-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-prereqs]
@@ -19,9 +20,101 @@ include::{es-ref-dir}/search/search-your-data/search-across-clusters.asciidoc[ta
 
 include::{es-ref-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-proxy-mode]
 
+[discrete]
+[[esql-ccs-security-model]]
+==== Security model
+
+{es} supports two security models for cross-cluster search (CCS):
+
+* <<esql-ccs-security-model-certificate, TLS certificate authentication>>
+* <<esql-ccs-security-model-api-key, API key authentication>>
+
+[TIP]
+====
+To check which security model is being used to connect your clusters, run `GET _remote/info`.
+If you're using the API key authentication method, you'll see the `"cluster_credentials"` key in the response.
+====
+
+[discrete]
+[[esql-ccs-security-model-certificate]]
+===== TLS certificate authentication
+
+TLS certificate authentication secures remote clusters with mutual TLS.
+This could be the preferred model when a single administrator has full control over both clusters.
+We generally recommend that roles and their privileges be identical in both clusters.
+
+Refer to <<remote-clusters-cert, TLS certificate authentication>> for prerequisites and detailed setup instructions.
+
+[discrete]
+[[esql-ccs-security-model-api-key]]
+===== API key authentication
+
+[NOTE]
+====
+`ENRICH` is *not supported* in this version when using {esql} with the API key based security model.
+====
+
+The following information pertains to using {esql} across clusters with the <<remote-clusters-api-key, *API key based security model*>>. You'll need to follow the steps on that page for the *full setup instructions*. This page only contains additional information specific to {esql}.
+
+API key based cross-cluster search (CCS) enables more granular control over allowed actions between clusters.
+This may be the preferred model when you have different administrators for different clusters and want more control over who can access what data. In this model, cluster administrators must explicitly define the access given to clusters and users.
+
+You will need to:
+
+* Create an API key on the *remote cluster* using the <<security-api-create-cross-cluster-api-key,Create cross-cluster API key>> API or using the {kibana-ref}/api-keys.html[Kibana API keys UI].
+* Add the API key to the keystore on the *local cluster*, as part of the steps in <<remote-clusters-security-api-key-local-actions,configuring the local cluster>>. All cross-cluster requests from the local cluster are bound by the API key’s privileges.
+
+Using {esql} with the API key based security model requires some additional permissions that may not be needed when using the traditional query DSL based search. 
+The following example API call creates a role that can query remote indices using {esql} when using the API key based security model.
+
+[source,console]
+----
+POST /_security/role/remote1
+{
+  "cluster": ["cross_cluster_search"], <1>
+  "indices": [
+    { 
+      "names" : [""], <2>
+      "privileges": ["read"]
+    }
+  ], 
+  "remote_indices": [ <3>
+    {
+      "names": [ "logs-*" ],
+      "privileges": [ "read","read_cross_cluster" ], <4>
+      "clusters" : ["my_remote_cluster"] <5>
+    }
+  ]
+}
+----
+
+<1> The `cross_cluster_search` cluster privilege is required for the _local_ cluster.
+<2> Typically, users will have permissions to read both local and remote indices. However, for cases where the role is intended to ONLY search the remote cluster, the `read` permission is still required for the local cluster. To provide read access to the local cluster, but disallow reading any indices in the local cluster, the `names` field may be an empty string.
+<3> The indices allowed read access to the remote cluster. The configured <<security-api-create-cross-cluster-api-key,cross-cluster API key>> must also allow this index to be read.
+<4> The `read_cross_cluster` privilege is always required when using {esql} across clusters with the API key based security model.
+<5> The remote clusters to which these privileges apply.
+This remote cluster must be configured with a <<security-api-create-cross-cluster-api-key,cross-cluster API key>> and connected to the remote cluster before the remote index can be queried. 
+Verify connection using the <<cluster-remote-info, Remote cluster info>> API.
+
+You will then need a user or API key with the permissions you created above. The following example API call creates a user with the `remote1` role.
+
+[source,console]
+----
+POST /_security/user/remote_user
+{
+  "password" : "<PASSWORD>",
+  "roles" : [ "remote1" ]
+}
+----
+
+Remember that all cross-cluster requests from the local cluster are bound by the cross cluster API key’s privileges, which are controlled by the remote cluster's administrator.
+
 [discrete]
 [[ccq-remote-cluster-setup]]
 ==== Remote cluster setup
+
+Once the security model is configured, you can add remote clusters.
+
 include::{es-ref-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-remote-cluster-setup]
 
 <1> Since `skip_unavailable` was not set on `cluster_three`, it uses
@@ -71,13 +164,18 @@ FROM *:my-index-000001
 Enrich in {esql} across clusters operates similarly to <<esql-enrich,local enrich>>.
 If the enrich policy and its enrich indices are consistent across all clusters, simply
 write the enrich command as you would without remote clusters. In this default mode,
-{esql} can execute the enrich command on either the querying cluster or the fulfilling
+{esql} can execute the enrich command on either the local cluster or the remote
 clusters, aiming to minimize computation or inter-cluster data transfer. Ensuring that
-the policy exists with consistent data on both the querying cluster and the fulfilling
+the policy exists with consistent data on both the local cluster and the remote
 clusters is critical for ES|QL to produce a consistent query result.
 
+[NOTE]
+====
+Enrich across clusters is *not supported* in this version when using {esql} with the <<remote-clusters-api-key, *API key based security model*>>.
+====
+
 In the following example, the enrich with `hosts` policy can be executed on
-either the querying cluster or the remote cluster `cluster_one`.
+either the local cluster or the remote cluster `cluster_one`.
 
 [source,esql]
 ----
@@ -87,8 +185,8 @@ FROM my-index-000001,cluster_one:my-index-000001
 ----
 
 Enrich with an {esql} query against remote clusters only can also happen on
-the querying cluster. This means the below query requires the `hosts` enrich
-policy to exist on the querying cluster as well.
+the local cluster. This means the below query requires the `hosts` enrich
+policy to exist on the local cluster as well.
 
 [source,esql]
 ----
@@ -99,10 +197,10 @@ FROM cluster_one:my-index-000001,cluster_two:my-index-000001
 
 [discrete]
 [[esql-enrich-coordinator]]
-==== Enrich with coordinator mode
+===== Enrich with coordinator mode
 
 {esql} provides the enrich `_coordinator` mode to force {esql} to execute the enrich
-command on the querying cluster. This mode should be used when the enrich policy is
+command on the local cluster. This mode should be used when the enrich policy is
 not available on the remote clusters or maintaining consistency of enrich indices
 across clusters is challenging.
 
@@ -118,21 +216,21 @@ FROM my-index-000001,cluster_one:my-index-000001
 [IMPORTANT]
 ====
 Enrich with the `_coordinator` mode usually increases inter-cluster data transfer and
-workload on the querying cluster.
+workload on the local cluster.
 ====
 
 [discrete]
 [[esql-enrich-remote]]
-==== Enrich with remote mode
+===== Enrich with remote mode
 
 {esql} also provides the enrich `_remote` mode to force {esql} to execute the enrich
-command independently on each fulfilling cluster where the target indices reside.
+command independently on each remote cluster where the target indices reside.
 This mode is useful for managing different enrich data on each cluster, such as detailed
 information of hosts for each region where the target (main) indices contain
 log events from these hosts.
 
 In the below example, the `hosts` enrich policy is required to exist on all
-fulfilling clusters: the `querying` cluster (as local indices are included),
+remote clusters: the `querying` cluster (as local indices are included),
 the remote cluster `cluster_one`, and `cluster_two`.
 
 [source,esql]
@@ -157,12 +255,12 @@ FROM my-index-000001,cluster_one:my-index-000001,cluster_two:my-index-000001
 
 [discrete]
 [[esql-multi-enrich]]
-==== Multiple enrich commands
+===== Multiple enrich commands
 
 You can include multiple enrich commands in the same query with different
 modes. {esql} will attempt to execute them accordingly. For example, this
 query performs two enriches, first with the `hosts` policy on any cluster
-and then with the `vendors` policy on the querying cluster.
+and then with the `vendors` policy on the local cluster.
 
 [source,esql]
 ----
diff --git a/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc b/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc
index 5f462b14405ba..4aa97ce375d9f 100644
--- a/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc
+++ b/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc
@@ -63,6 +63,7 @@ information, refer to https://www.elastic.co/subscriptions.
 NOTE: If a remote cluster is part of an {ess} deployment, it has a valid certificate by default. 
 You can therefore skip steps related to certificates in these instructions.
 
+[[remote-clusters-security-api-key-remote-action]]
 ===== On the remote cluster
 
 // tag::remote-cluster-steps[]
@@ -155,6 +156,7 @@ to the indices you want to use for {ccs} or {ccr}. You can use the
 need it to connect to the remote cluster later.
 // end::remote-cluster-steps[]
 
+[[remote-clusters-security-api-key-local-actions]]
 ===== On the local cluster
 
 // tag::local-cluster-steps[]

From bf8471c264d5d63d99c6bc05d4a364ab16cea6ab Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Mon, 3 Jun 2024 13:16:19 -0400
Subject: [PATCH 119/208] [ESQL] Migrate SimplifyComparisonArithmetics
 optimization (#109256)

This pulls the SimplifyComparisonArithmetics optimization into ESQL proper, from core, and in doing so resolves two of the issues we uncovered with that optimization.

Rather than moving this into a sub-class of OptimzierRules, I've opted to create a new package optimizer.rules, and put this as a top level class there. I have another PR open that migrates a bunch of other rules into OptimizerRules, and that's in general an area of the code that's being touched a lot lately. Having top level classes in a package, rather than sub-classes in a single class file, will create fewer conflicts while working on things that are actually unrelated in this type of situation.

Although this is a bugfix, I did not mark it for backporting to 8.14. To my knowledge, no users have reported the associated bugs (we found them through migrating tests), and this fix depends on tests and refactorings that were not backported to 8.14. It would be possible to write an 8.14 version of this, but IMHO it's easier to do as a fresh ticket than as a backport and conflict resolution process. If we think that's important, I'm happy to do so, but at the moment I think fixing it for 8.15 is fine.

Note that this is a re-application of the changes in #108200 after we did other refactoring, which that was closed to enable.

Resolves #108388
Resolves #108743
---
 docs/changelog/109256.yaml                    |   7 +
 .../esql/core/optimizer/OptimizerRules.java   | 224 ----------------
 .../esql/optimizer/LogicalPlanOptimizer.java  |   2 +-
 .../rules/SimplifyComparisonsArithmetics.java | 244 ++++++++++++++++++
 .../optimizer/LogicalPlanOptimizerTests.java  |   4 -
 5 files changed, 252 insertions(+), 229 deletions(-)
 create mode 100644 docs/changelog/109256.yaml
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java

diff --git a/docs/changelog/109256.yaml b/docs/changelog/109256.yaml
new file mode 100644
index 0000000000000..30c15ed77f9b9
--- /dev/null
+++ b/docs/changelog/109256.yaml
@@ -0,0 +1,7 @@
+pr: 109256
+summary: "[ESQL] Migrate `SimplifyComparisonArithmetics` optimization"
+area: ES|QL
+type: bug
+issues:
+ - 108388
+ - 108743
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java
index 4ce5f4ae8652e..12b496e51fa1b 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java
@@ -29,10 +29,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.ArithmeticOperation;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.Neg;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.Sub;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.GreaterThan;
@@ -49,12 +45,10 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.rule.Rule;
-import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.ReflectionUtils;
 
-import java.time.DateTimeException;
 import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -66,8 +60,6 @@
 import java.util.Set;
 import java.util.function.BiFunction;
 
-import static java.lang.Math.signum;
-import static java.util.Arrays.asList;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
@@ -77,12 +69,6 @@
 import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitAnd;
 import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.splitOr;
 import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.subtract;
-import static org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation.ADD;
-import static org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation.DIV;
-import static org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation.MOD;
-import static org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation.MUL;
-import static org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation.SUB;
-import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.esql.core.util.CollectionUtils.combine;
 
 public final class OptimizerRules {
@@ -1282,216 +1268,6 @@ protected Expression rule(Expression e) {
         }
     }
 
-    // Simplifies arithmetic expressions with BinaryComparisons and fixed point fields, such as: (int + 2) / 3 > 4 => int > 10
-    public static final class SimplifyComparisonsArithmetics extends OptimizerExpressionRule<BinaryComparison> {
-        BiFunction<DataType, DataType, Boolean> typesCompatible;
-
-        public SimplifyComparisonsArithmetics(BiFunction<DataType, DataType, Boolean> typesCompatible) {
-            super(TransformDirection.UP);
-            this.typesCompatible = typesCompatible;
-        }
-
-        @Override
-        protected Expression rule(BinaryComparison bc) {
-            // optimize only once the expression has a literal on the right side of the binary comparison
-            if (bc.right() instanceof Literal) {
-                if (bc.left() instanceof ArithmeticOperation) {
-                    return simplifyBinaryComparison(bc);
-                }
-                if (bc.left() instanceof Neg) {
-                    return foldNegation(bc);
-                }
-            }
-            return bc;
-        }
-
-        private Expression simplifyBinaryComparison(BinaryComparison comparison) {
-            ArithmeticOperation operation = (ArithmeticOperation) comparison.left();
-            // Use symbol comp: SQL operations aren't available in this package (as dependencies)
-            String opSymbol = operation.symbol();
-            // Modulo can't be simplified.
-            if (opSymbol.equals(MOD.symbol())) {
-                return comparison;
-            }
-            OperationSimplifier simplification = null;
-            if (isMulOrDiv(opSymbol)) {
-                simplification = new MulDivSimplifier(comparison);
-            } else if (opSymbol.equals(ADD.symbol()) || opSymbol.equals(SUB.symbol())) {
-                simplification = new AddSubSimplifier(comparison);
-            }
-
-            return (simplification == null || simplification.isUnsafe(typesCompatible)) ? comparison : simplification.apply();
-        }
-
-        private static boolean isMulOrDiv(String opSymbol) {
-            return opSymbol.equals(MUL.symbol()) || opSymbol.equals(DIV.symbol());
-        }
-
-        private static Expression foldNegation(BinaryComparison bc) {
-            Literal bcLiteral = (Literal) bc.right();
-            Expression literalNeg = tryFolding(new Neg(bcLiteral.source(), bcLiteral));
-            return literalNeg == null ? bc : bc.reverse().replaceChildren(asList(((Neg) bc.left()).field(), literalNeg));
-        }
-
-        private static Expression tryFolding(Expression expression) {
-            if (expression.foldable()) {
-                try {
-                    expression = new Literal(expression.source(), expression.fold(), expression.dataType());
-                } catch (ArithmeticException | DateTimeException e) {
-                    // null signals that folding would result in an over-/underflow (such as Long.MAX_VALUE+1); the optimisation is skipped.
-                    expression = null;
-                }
-            }
-            return expression;
-        }
-
-        private abstract static class OperationSimplifier {
-            final BinaryComparison comparison;
-            final Literal bcLiteral;
-            final ArithmeticOperation operation;
-            final Expression opLeft;
-            final Expression opRight;
-            final Literal opLiteral;
-
-            OperationSimplifier(BinaryComparison comparison) {
-                this.comparison = comparison;
-                operation = (ArithmeticOperation) comparison.left();
-                bcLiteral = (Literal) comparison.right();
-
-                opLeft = operation.left();
-                opRight = operation.right();
-
-                if (opLeft instanceof Literal) {
-                    opLiteral = (Literal) opLeft;
-                } else if (opRight instanceof Literal) {
-                    opLiteral = (Literal) opRight;
-                } else {
-                    opLiteral = null;
-                }
-            }
-
-            // can it be quickly fast-tracked that the operation can't be reduced?
-            final boolean isUnsafe(BiFunction<DataType, DataType, Boolean> typesCompatible) {
-                if (opLiteral == null) {
-                    // one of the arithm. operands must be a literal, otherwise the operation wouldn't simplify anything
-                    return true;
-                }
-
-                // Only operations on fixed point literals are supported, since optimizing float point operations can also change the
-                // outcome of the filtering:
-                // x + 1e18 > 1e18::long will yield different results with a field value in [-2^6, 2^6], optimised vs original;
-                // x * (1 + 1e-15d) > 1 : same with a field value of (1 - 1e-15d)
-                // so consequently, int fields optimisation requiring FP arithmetic isn't possible either: (x - 1e-15) * (1 + 1e-15) > 1.
-                if (opLiteral.dataType().isRational() || bcLiteral.dataType().isRational()) {
-                    return true;
-                }
-
-                // the Literal will be moved to the right of the comparison, but only if data-compatible with what's there
-                if (typesCompatible.apply(bcLiteral.dataType(), opLiteral.dataType()) == false) {
-                    return true;
-                }
-
-                return isOpUnsafe();
-            }
-
-            final Expression apply() {
-                // force float point folding for FlP field
-                Literal bcl = operation.dataType().isRational()
-                    ? Literal.of(bcLiteral, ((Number) bcLiteral.value()).doubleValue())
-                    : bcLiteral;
-
-                Expression bcRightExpression = ((BinaryComparisonInversible) operation).binaryComparisonInverse()
-                    .create(bcl.source(), bcl, opRight);
-                bcRightExpression = tryFolding(bcRightExpression);
-                return bcRightExpression != null
-                    ? postProcess((BinaryComparison) comparison.replaceChildren(List.of(opLeft, bcRightExpression)))
-                    : comparison;
-            }
-
-            // operation-specific operations:
-            // - fast-tracking of simplification unsafety
-            abstract boolean isOpUnsafe();
-
-            // - post optimisation adjustments
-            Expression postProcess(BinaryComparison binaryComparison) {
-                return binaryComparison;
-            }
-        }
-
-        private static class AddSubSimplifier extends OperationSimplifier {
-
-            AddSubSimplifier(BinaryComparison comparison) {
-                super(comparison);
-            }
-
-            @Override
-            boolean isOpUnsafe() {
-                // no ADD/SUB with floating fields
-                if (operation.dataType().isRational()) {
-                    return true;
-                }
-
-                if (operation.symbol().equals(SUB.symbol()) && opRight instanceof Literal == false) { // such as: 1 - x > -MAX
-                    // if next simplification step would fail on overflow anyways, skip the optimisation already
-                    return tryFolding(new Sub(EMPTY, opLeft, bcLiteral)) == null;
-                }
-
-                return false;
-            }
-        }
-
-        private static class MulDivSimplifier extends OperationSimplifier {
-
-            private final boolean isDiv; // and not MUL.
-            private final int opRightSign; // sign of the right operand in: (left) (op) (right) (comp) (literal)
-
-            MulDivSimplifier(BinaryComparison comparison) {
-                super(comparison);
-                isDiv = operation.symbol().equals(DIV.symbol());
-                opRightSign = sign(opRight);
-            }
-
-            @Override
-            boolean isOpUnsafe() {
-                // Integer divisions are not safe to optimise: x / 5 > 1 <=/=> x > 5 for x in [6, 9]; same for the `==` comp
-                if (operation.dataType().isInteger() && isDiv) {
-                    return true;
-                }
-
-                // If current operation is a multiplication, it's inverse will be a division: safe only if outcome is still integral.
-                if (isDiv == false && opLeft.dataType().isInteger()) {
-                    long opLiteralValue = ((Number) opLiteral.value()).longValue();
-                    return opLiteralValue == 0 || ((Number) bcLiteral.value()).longValue() % opLiteralValue != 0;
-                }
-
-                // can't move a 0 in Mul/Div comparisons
-                return opRightSign == 0;
-            }
-
-            @Override
-            Expression postProcess(BinaryComparison binaryComparison) {
-                // negative multiplication/division changes the direction of the comparison
-                return opRightSign < 0 ? binaryComparison.reverse() : binaryComparison;
-            }
-
-            private static int sign(Object obj) {
-                int sign = 1;
-                if (obj instanceof Number) {
-                    sign = (int) signum(((Number) obj).doubleValue());
-                } else if (obj instanceof Literal) {
-                    sign = sign(((Literal) obj).value());
-                } else if (obj instanceof Neg) {
-                    sign = -sign(((Neg) obj).field());
-                } else if (obj instanceof ArithmeticOperation operation) {
-                    if (isMulOrDiv(operation.symbol())) {
-                        sign = sign(operation.left()) * sign(operation.right());
-                    }
-                }
-                return sign;
-            }
-        }
-    }
-
     public abstract static class PruneFilters extends OptimizerRule<Filter> {
 
         @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
index d97d54eb884f3..7beb3aca05e74 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.LiteralsOnTheRight;
 import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.PruneLiteralsInOrderBy;
 import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.SetAsOptimized;
-import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.SimplifyComparisonsArithmetics;
 import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.core.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
@@ -62,6 +61,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
+import org.elasticsearch.xpack.esql.optimizer.rules.SimplifyComparisonsArithmetics;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java
new file mode 100644
index 0000000000000..9a7ee0a587335
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java
@@ -0,0 +1,244 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.optimizer.rules;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.ArithmeticOperation;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub;
+
+import java.time.DateTimeException;
+import java.util.List;
+import java.util.function.BiFunction;
+
+import static java.lang.Math.signum;
+import static java.util.Arrays.asList;
+import static org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation.ADD;
+import static org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation.DIV;
+import static org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation.MOD;
+import static org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation.MUL;
+import static org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation.SUB;
+import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
+
+/**
+ * Simplifies arithmetic expressions with BinaryComparisons and fixed point fields, such as: (int + 2) / 3 > 4 => int > 10
+ */
+public final class SimplifyComparisonsArithmetics extends
+    org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.OptimizerExpressionRule<BinaryComparison> {
+    BiFunction<DataType, DataType, Boolean> typesCompatible;
+
+    public SimplifyComparisonsArithmetics(BiFunction<DataType, DataType, Boolean> typesCompatible) {
+        super(org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.TransformDirection.UP);
+        this.typesCompatible = typesCompatible;
+    }
+
+    @Override
+    protected Expression rule(BinaryComparison bc) {
+        // optimize only once the expression has a literal on the right side of the binary comparison
+        if (bc.right() instanceof Literal) {
+            if (bc.left() instanceof ArithmeticOperation) {
+                return simplifyBinaryComparison(bc);
+            }
+            if (bc.left() instanceof Neg) {
+                return foldNegation(bc);
+            }
+        }
+        return bc;
+    }
+
+    private Expression simplifyBinaryComparison(BinaryComparison comparison) {
+        ArithmeticOperation operation = (ArithmeticOperation) comparison.left();
+        // Use symbol comp: SQL operations aren't available in this package (as dependencies)
+        String opSymbol = operation.symbol();
+        // Modulo can't be simplified.
+        if (opSymbol.equals(MOD.symbol())) {
+            return comparison;
+        }
+        OperationSimplifier simplification = null;
+        if (isMulOrDiv(opSymbol)) {
+            simplification = new MulDivSimplifier(comparison);
+        } else if (opSymbol.equals(ADD.symbol()) || opSymbol.equals(SUB.symbol())) {
+            simplification = new AddSubSimplifier(comparison);
+        }
+
+        return (simplification == null || simplification.isUnsafe(typesCompatible)) ? comparison : simplification.apply();
+    }
+
+    private static boolean isMulOrDiv(String opSymbol) {
+        return opSymbol.equals(MUL.symbol()) || opSymbol.equals(DIV.symbol());
+    }
+
+    private static Expression foldNegation(BinaryComparison bc) {
+        Literal bcLiteral = (Literal) bc.right();
+        Expression literalNeg = tryFolding(new Neg(bcLiteral.source(), bcLiteral));
+        return literalNeg == null ? bc : bc.reverse().replaceChildren(asList(((Neg) bc.left()).field(), literalNeg));
+    }
+
+    private static Expression tryFolding(Expression expression) {
+        if (expression.foldable()) {
+            try {
+                expression = new Literal(expression.source(), expression.fold(), expression.dataType());
+            } catch (ArithmeticException | DateTimeException e) {
+                // null signals that folding would result in an over-/underflow (such as Long.MAX_VALUE+1); the optimisation is skipped.
+                expression = null;
+            }
+        }
+        return expression;
+    }
+
+    private abstract static class OperationSimplifier {
+        final BinaryComparison comparison;
+        final Literal bcLiteral;
+        final ArithmeticOperation operation;
+        final Expression opLeft;
+        final Expression opRight;
+        final Literal opLiteral;
+
+        OperationSimplifier(BinaryComparison comparison) {
+            this.comparison = comparison;
+            operation = (ArithmeticOperation) comparison.left();
+            bcLiteral = (Literal) comparison.right();
+
+            opLeft = operation.left();
+            opRight = operation.right();
+
+            if (opLeft instanceof Literal) {
+                opLiteral = (Literal) opLeft;
+            } else if (opRight instanceof Literal) {
+                opLiteral = (Literal) opRight;
+            } else {
+                opLiteral = null;
+            }
+        }
+
+        // can it be quickly fast-tracked that the operation can't be reduced?
+        final boolean isUnsafe(BiFunction<DataType, DataType, Boolean> typesCompatible) {
+            if (opLiteral == null) {
+                // one of the arithm. operands must be a literal, otherwise the operation wouldn't simplify anything
+                return true;
+            }
+
+            // Only operations on fixed point literals are supported, since optimizing float point operations can also change the
+            // outcome of the filtering:
+            // x + 1e18 > 1e18::long will yield different results with a field value in [-2^6, 2^6], optimised vs original;
+            // x * (1 + 1e-15d) > 1 : same with a field value of (1 - 1e-15d)
+            // so consequently, int fields optimisation requiring FP arithmetic isn't possible either: (x - 1e-15) * (1 + 1e-15) > 1.
+            if (opLiteral.dataType().isRational() || bcLiteral.dataType().isRational()) {
+                return true;
+            }
+
+            // the Literal will be moved to the right of the comparison, but only if data-compatible with what's there
+            if (typesCompatible.apply(bcLiteral.dataType(), opLiteral.dataType()) == false) {
+                return true;
+            }
+
+            return isOpUnsafe();
+        }
+
+        final Expression apply() {
+            // force float point folding for FlP field
+            Literal bcl = operation.dataType().isRational()
+                ? new Literal(bcLiteral.source(), ((Number) bcLiteral.value()).doubleValue(), DataTypes.DOUBLE)
+                : bcLiteral;
+
+            Expression bcRightExpression = ((BinaryComparisonInversible) operation).binaryComparisonInverse()
+                .create(bcl.source(), bcl, opRight);
+            bcRightExpression = tryFolding(bcRightExpression);
+            return bcRightExpression != null
+                ? postProcess((BinaryComparison) comparison.replaceChildren(List.of(opLeft, bcRightExpression)))
+                : comparison;
+        }
+
+        // operation-specific operations:
+        // - fast-tracking of simplification unsafety
+        abstract boolean isOpUnsafe();
+
+        // - post optimisation adjustments
+        Expression postProcess(BinaryComparison binaryComparison) {
+            return binaryComparison;
+        }
+    }
+
+    private static class AddSubSimplifier extends OperationSimplifier {
+
+        AddSubSimplifier(BinaryComparison comparison) {
+            super(comparison);
+        }
+
+        @Override
+        boolean isOpUnsafe() {
+            // no ADD/SUB with floating fields
+            if (operation.dataType().isRational()) {
+                return true;
+            }
+
+            if (operation.symbol().equals(SUB.symbol()) && opRight instanceof Literal == false) { // such as: 1 - x > -MAX
+                // if next simplification step would fail on overflow anyways, skip the optimisation already
+                return tryFolding(new Sub(EMPTY, opLeft, bcLiteral)) == null;
+            }
+
+            return false;
+        }
+    }
+
+    private static class MulDivSimplifier extends OperationSimplifier {
+
+        private final boolean isDiv; // and not MUL.
+        private final int opRightSign; // sign of the right operand in: (left) (op) (right) (comp) (literal)
+
+        MulDivSimplifier(BinaryComparison comparison) {
+            super(comparison);
+            isDiv = operation.symbol().equals(DIV.symbol());
+            opRightSign = sign(opRight);
+        }
+
+        @Override
+        boolean isOpUnsafe() {
+            // Integer divisions are not safe to optimise: x / 5 > 1 <=/=> x > 5 for x in [6, 9]; same for the `==` comp
+            if (operation.dataType().isInteger() && isDiv) {
+                return true;
+            }
+
+            // If current operation is a multiplication, it's inverse will be a division: safe only if outcome is still integral.
+            if (isDiv == false && opLeft.dataType().isInteger()) {
+                long opLiteralValue = ((Number) opLiteral.value()).longValue();
+                return opLiteralValue == 0 || ((Number) bcLiteral.value()).longValue() % opLiteralValue != 0;
+            }
+
+            // can't move a 0 in Mul/Div comparisons
+            return opRightSign == 0;
+        }
+
+        @Override
+        Expression postProcess(BinaryComparison binaryComparison) {
+            // negative multiplication/division changes the direction of the comparison
+            return opRightSign < 0 ? binaryComparison.reverse() : binaryComparison;
+        }
+
+        private static int sign(Object obj) {
+            int sign = 1;
+            if (obj instanceof Number) {
+                sign = (int) signum(((Number) obj).doubleValue());
+            } else if (obj instanceof Literal) {
+                sign = sign(((Literal) obj).value());
+            } else if (obj instanceof Neg) {
+                sign = -sign(((Neg) obj).field());
+            } else if (obj instanceof ArithmeticOperation operation) {
+                if (isMulOrDiv(operation.symbol())) {
+                    sign = sign(operation.left()) * sign(operation.right());
+                }
+            }
+            return sign;
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
index cfe2535519557..4bb797faff04c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
@@ -4554,7 +4554,6 @@ public void testSimplifyComparisonArithmeticCommutativeVsNonCommutativeOps() {
 
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108388")
     public void testSimplifyComparisonArithmeticsWithFloatingPoints() {
         doTestSimplifyComparisonArithmetics("float / 2 > 4", "float", GT, 8d);
     }
@@ -4578,17 +4577,14 @@ public void testSimplifyComparisonArithmeticWithMultipleOps() {
         doTestSimplifyComparisonArithmetics("((integer + 1) * 2 - 4) * 4 >= 16", "integer", GTE, 3);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108743")
     public void testSimplifyComparisonArithmeticWithFieldNegation() {
         doTestSimplifyComparisonArithmetics("12 * (-integer - 5) >= -120", "integer", LTE, 5);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108743")
     public void testSimplifyComparisonArithmeticWithFieldDoubleNegation() {
         doTestSimplifyComparisonArithmetics("12 * -(-integer - 5) <= 120", "integer", LTE, 5);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108743")
     public void testSimplifyComparisonArithmeticWithConjunction() {
         doTestSimplifyComparisonArithmetics("12 * (-integer - 5) == -120 AND integer < 6 ", "integer", EQ, 5);
     }

From 88cd264c7fc6986bc06a2ce7942c74be759cce31 Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Mon, 3 Jun 2024 12:14:50 -0700
Subject: [PATCH 120/208] Add metrics for mappings incompatible with synthetic
 source (#109268)

---
 .../index/mapper/DocumentMapper.java          |  8 +-
 .../index/mapper/SourceFieldMetrics.java      | 12 +++
 ...ests.java => SourceFieldMetricsTests.java} | 17 +++-
 .../index/mapper/MapperServiceTestCase.java   | 89 ++++++++++++++-----
 4 files changed, 99 insertions(+), 27 deletions(-)
 rename server/src/test/java/org/elasticsearch/index/mapper/{SourceLoaderTelemetryTests.java => SourceFieldMetricsTests.java} (70%)

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
index 1b07d93295fe1..0136175cc6391 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
@@ -126,7 +126,13 @@ public void validate(IndexSettings settings, boolean checkLimits) {
          * Build an empty source loader to validate that the mapping is compatible
          * with the source loading strategy declared on the source field mapper.
          */
-        sourceMapper().newSourceLoader(mapping(), mapperMetrics.sourceFieldMetrics());
+        try {
+            sourceMapper().newSourceLoader(mapping(), mapperMetrics.sourceFieldMetrics());
+        } catch (IllegalArgumentException e) {
+            mapperMetrics.sourceFieldMetrics().recordSyntheticSourceIncompatibleMapping();
+            throw e;
+        }
+
         if (settings.getIndexSortConfig().hasIndexSort() && mappers().nestedLookup() != NestedLookup.EMPTY) {
             throw new IllegalArgumentException("cannot have nested fields when index sort is activated");
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java
index 0e6ce79fd2170..eaccdbc9e44ce 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMetrics.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.telemetry.metric.LongCounter;
 import org.elasticsearch.telemetry.metric.LongHistogram;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 
@@ -21,10 +22,12 @@ public class SourceFieldMetrics {
     public static final SourceFieldMetrics NOOP = new SourceFieldMetrics(MeterRegistry.NOOP, () -> 0);
 
     public static final String SYNTHETIC_SOURCE_LOAD_LATENCY = "es.mapper.synthetic_source.load.latency.histogram";
+    public static final String SYNTHETIC_SOURCE_INCOMPATIBLE_MAPPING = "es.mapper.synthetic_source.incompatible_mapping.total";
 
     private final LongSupplier relativeTimeSupplier;
 
     private final LongHistogram syntheticSourceLoadLatency;
+    private final LongCounter syntheticSourceIncompatibleMapping;
 
     public SourceFieldMetrics(MeterRegistry meterRegistry, LongSupplier relativeTimeSupplier) {
         this.syntheticSourceLoadLatency = meterRegistry.registerLongHistogram(
@@ -32,6 +35,11 @@ public SourceFieldMetrics(MeterRegistry meterRegistry, LongSupplier relativeTime
             "Time it takes to load fields and construct synthetic source",
             "ms"
         );
+        this.syntheticSourceIncompatibleMapping = meterRegistry.registerLongCounter(
+            SYNTHETIC_SOURCE_INCOMPATIBLE_MAPPING,
+            "Number of create/update index operations using mapping not compatible with synthetic source",
+            "count"
+        );
         this.relativeTimeSupplier = relativeTimeSupplier;
     }
 
@@ -42,4 +50,8 @@ public LongSupplier getRelativeTimeSupplier() {
     public void recordSyntheticSourceLoadLatency(TimeValue value) {
         this.syntheticSourceLoadLatency.record(value.millis());
     }
+
+    public void recordSyntheticSourceIncompatibleMapping() {
+        this.syntheticSourceIncompatibleMapping.increment();
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java
similarity index 70%
rename from server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java
rename to server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java
index 1c88cbb0d8592..f569a69246d9f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTelemetryTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java
@@ -20,7 +20,7 @@
 import java.util.Collection;
 import java.util.List;
 
-public class SourceLoaderTelemetryTests extends MapperServiceTestCase {
+public class SourceFieldMetricsTests extends MapperServiceTestCase {
     private final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin();
 
     @Override
@@ -34,8 +34,8 @@ public void testFieldHasValue() {}
     @Override
     public void testFieldHasValueWithEmptyFieldInfos() {}
 
-    public void testSyntheticSourceTelemetry() throws IOException {
-        var mapping = syntheticSourceMapping(b -> { b.startObject("kwd").field("type", "keyword").endObject(); });
+    public void testSyntheticSourceLoadLatency() throws IOException {
+        var mapping = syntheticSourceMapping(b -> b.startObject("kwd").field("type", "keyword").endObject());
         var mapper = createDocumentMapper(mapping);
 
         try (Directory directory = newDirectory()) {
@@ -58,4 +58,15 @@ public void testSyntheticSourceTelemetry() throws IOException {
         // test implementation of time provider always has a gap of 1 between values
         assertEquals(measurements.get(0).getLong(), 1);
     }
+
+    public void testSyntheticSourceIncompatibleMapping() throws IOException {
+        var mapping = syntheticSourceMapping(b -> b.startObject("kwd").field("type", "text").field("store", "false").endObject());
+        var mapperMetrics = createTestMapperMetrics();
+        var mapperService = new TestMapperServiceBuilder().mapperMetrics(mapperMetrics).build();
+        assertThrows(IllegalArgumentException.class, () -> withMapping(mapperService, mapping));
+
+        var measurements = telemetryPlugin.getLongCounterMeasurement(SourceFieldMetrics.SYNTHETIC_SOURCE_INCOMPATIBLE_MAPPING);
+        assertEquals(1, measurements.size());
+        assertEquals(measurements.get(0).getLong(), 1);
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
index 791f6230933ad..7d13e33be0db7 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
@@ -61,6 +61,7 @@
 import org.elasticsearch.plugins.TelemetryPlugin;
 import org.elasticsearch.plugins.internal.DocumentSizeObserver;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -198,34 +199,76 @@ protected final MapperService createMapperService(
         BooleanSupplier idFieldDataEnabled,
         XContentBuilder mapping
     ) throws IOException {
-
         MapperService mapperService = createMapperService(version, settings, idFieldDataEnabled);
-        merge(mapperService, mapping);
-        return mapperService;
+        return withMapping(mapperService, mapping);
     }
 
     protected final MapperService createMapperService(IndexVersion version, Settings settings, BooleanSupplier idFieldDataEnabled) {
-        IndexSettings indexSettings = createIndexSettings(version, settings);
-        MapperRegistry mapperRegistry = new IndicesModule(
-            getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList())
-        ).getMapperRegistry();
-
-        SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
-        return new MapperService(
-            () -> TransportVersion.current(),
-            indexSettings,
-            createIndexAnalyzers(indexSettings),
-            parserConfig(),
-            similarityService,
-            mapperRegistry,
-            () -> {
-                throw new UnsupportedOperationException();
-            },
-            indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled),
-            this::compileScript,
-            MapperMetrics.NOOP
+        return new TestMapperServiceBuilder().indexVersion(version).settings(settings).idFieldDataEnabled(idFieldDataEnabled).build();
+    }
 
-        );
+    protected final MapperService withMapping(MapperService mapperService, XContentBuilder mapping) throws IOException {
+        merge(mapperService, mapping);
+        return mapperService;
+    };
+
+    protected class TestMapperServiceBuilder {
+        private IndexVersion indexVersion;
+        private Settings settings;
+        private BooleanSupplier idFieldDataEnabled;
+        private ScriptCompiler scriptCompiler;
+        private MapperMetrics mapperMetrics;
+
+        public TestMapperServiceBuilder() {
+            indexVersion = getVersion();
+            settings = getIndexSettings();
+            idFieldDataEnabled = () -> true;
+            scriptCompiler = MapperServiceTestCase.this::compileScript;
+            mapperMetrics = MapperMetrics.NOOP;
+        }
+
+        public TestMapperServiceBuilder indexVersion(IndexVersion indexVersion) {
+            this.indexVersion = indexVersion;
+            return this;
+        }
+
+        public TestMapperServiceBuilder settings(Settings settings) {
+            this.settings = settings;
+            return this;
+        }
+
+        public TestMapperServiceBuilder idFieldDataEnabled(BooleanSupplier idFieldDataEnabled) {
+            this.idFieldDataEnabled = idFieldDataEnabled;
+            return this;
+        }
+
+        public TestMapperServiceBuilder mapperMetrics(MapperMetrics mapperMetrics) {
+            this.mapperMetrics = mapperMetrics;
+            return this;
+        }
+
+        public MapperService build() {
+            IndexSettings indexSettings = createIndexSettings(indexVersion, settings);
+            SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
+            MapperRegistry mapperRegistry = new IndicesModule(
+                getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList())
+            ).getMapperRegistry();
+
+            return new MapperService(
+                () -> TransportVersion.current(),
+                indexSettings,
+                createIndexAnalyzers(indexSettings),
+                parserConfig(),
+                similarityService,
+                mapperRegistry,
+                () -> {
+                    throw new UnsupportedOperationException();
+                },
+                indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled),
+                scriptCompiler,
+                mapperMetrics
+            );
+        }
     }
 
     /**

From 456e36e68b2aee890634378a91815e7bcaff6471 Mon Sep 17 00:00:00 2001
From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com>
Date: Mon, 3 Jun 2024 16:19:21 -0400
Subject: [PATCH 121/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109315

---
 muted-tests.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index d7d47a48fee23..ba2d1ce86f492 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -19,7 +19,8 @@ tests:
   method: "testGuessIsDayFirstFromLocale"
 - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108857"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
+    \ dependent mappings / dates}"
 - class: "org.elasticsearch.upgrades.SearchStatesIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108991"
   method: "testCanMatch"
@@ -28,7 +29,8 @@ tests:
   method: "testTrainedModelInference"
 - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109188"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
+    \ dependent mappings / dates}"
 - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109189"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
@@ -44,6 +46,9 @@ tests:
 - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
   method: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/109266
+- class: "org.elasticsearch.index.engine.frozen.FrozenIndexIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109315"
+  method: "testTimestampFieldTypeExposedByAllIndicesServices"
 
 # Examples:
 #

From 147dd0f3ca813cc387e4fd08a7b0f1c6e1a68849 Mon Sep 17 00:00:00 2001
From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com>
Date: Mon, 3 Jun 2024 16:36:00 -0400
Subject: [PATCH 122/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109318

---
 muted-tests.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index ba2d1ce86f492..4b09d4e8ba7ba 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -49,6 +49,8 @@ tests:
 - class: "org.elasticsearch.index.engine.frozen.FrozenIndexIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109315"
   method: "testTimestampFieldTypeExposedByAllIndicesServices"
+- class: "org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109318"
 
 # Examples:
 #

From 5c711d0c5b76c871ff113dfbaeb369f1411bcad0 Mon Sep 17 00:00:00 2001
From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com>
Date: Mon, 3 Jun 2024 17:09:46 -0400
Subject: [PATCH 123/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109322

---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index 4b09d4e8ba7ba..fd1999d201bf0 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -51,6 +51,9 @@ tests:
   method: "testTimestampFieldTypeExposedByAllIndicesServices"
 - class: "org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109318"
+- class: "org.elasticsearch.upgrades.AggregationsIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109322"
+  method: "testTerms"
 
 # Examples:
 #

From 3a4dfb00666a9c3d89b83fa24dec00c5320fb8d9 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 3 Jun 2024 18:23:13 -0700
Subject: [PATCH 124/208] Add remove index setting command (#109276)

The new subcommand elasticsearch-node remove-index-settings can be used
to remove index settings from the cluster state in case where it
contains incompatible index settings that prevent the cluster from
forming. This tool can cause data loss and its use should be your last
resort.

Relates #96075
---
 docs/changelog/109276.yaml                    |   5 +
 docs/reference/commands/node-tool.asciidoc    |  55 ++++++
 .../RemoveIndexSettingsCommandIT.java         | 162 ++++++++++++++++++
 .../cluster/coordination/NodeToolCli.java     |   1 +
 .../RemoveIndexSettingsCommand.java           | 100 +++++++++++
 5 files changed, 323 insertions(+)
 create mode 100644 docs/changelog/109276.yaml
 create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommand.java

diff --git a/docs/changelog/109276.yaml b/docs/changelog/109276.yaml
new file mode 100644
index 0000000000000..d73e68e3c8f7b
--- /dev/null
+++ b/docs/changelog/109276.yaml
@@ -0,0 +1,5 @@
+pr: 109276
+summary: Add remove index setting command
+area: Infra/Settings
+type: enhancement
+issues: []
diff --git a/docs/reference/commands/node-tool.asciidoc b/docs/reference/commands/node-tool.asciidoc
index 810de4a71fffb..cdd2bb8f0f9d7 100644
--- a/docs/reference/commands/node-tool.asciidoc
+++ b/docs/reference/commands/node-tool.asciidoc
@@ -31,6 +31,10 @@ This tool has a number of modes:
   from the cluster state in case where it contains incompatible settings that
   prevent the cluster from forming.
 
+* `elasticsearch-node remove-index-settings` can be used to remove index settings
+  from the cluster state in case where it contains incompatible index settings that
+  prevent the cluster from forming.
+
 * `elasticsearch-node remove-customs` can be used to remove custom metadata
   from the cluster state in case where it contains broken metadata that
   prevents the cluster state from being loaded.
@@ -107,6 +111,26 @@ The intended use is:
 * Repeat for all other master-eligible nodes
 * Start the nodes
 
+[discrete]
+==== Removing index settings
+
+There may be situations where an index contains index settings
+that prevent the cluster from forming. Since the cluster cannot form,
+it is not possible to remove these settings using the
+<<indices-update-settings>> API.
+
+The `elasticsearch-node remove-index-settings` tool allows you to forcefully remove
+those index settings from the on-disk cluster state. The tool takes a
+list of index settings as parameters that should be removed, and also supports
+wildcard patterns.
+
+The intended use is:
+
+* Stop the node
+* Run `elasticsearch-node remove-index-settings name-of-index-setting-to-remove` on the node
+* Repeat for all nodes
+* Start the nodes
+
 [discrete]
 ==== Removing custom metadata from the cluster state
 
@@ -436,6 +460,37 @@ You can also use wildcards to remove multiple settings, for example using
 node$ ./bin/elasticsearch-node remove-settings xpack.monitoring.*
 ----
 
+[discrete]
+==== Removing index settings
+
+If your indices contain index settings that prevent the cluster
+from forming, you can run the following command to remove one
+or more index settings.
+
+[source,txt]
+----
+node$ ./bin/elasticsearch-node remove-index-settings index.my_plugin.foo
+
+    WARNING: Elasticsearch MUST be stopped before running this tool.
+
+You should only run this tool if you have incompatible index settings in the
+cluster state that prevent the cluster from forming.
+This tool can cause data loss and its use should be your last resort.
+
+Do you want to proceed?
+
+Confirm [y/N] y
+
+Index settings were successfully removed from the cluster state
+----
+
+You can also use wildcards to remove multiple index settings, for example using
+
+[source,txt]
+----
+node$ ./bin/elasticsearch-node remove-index-settings index.my_plugin.*
+----
+
 [discrete]
 ==== Removing custom metadata from the cluster state
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java
new file mode 100644
index 0000000000000..a5e445270ccc4
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java
@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import joptsimple.OptionSet;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.ProcessInfo;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+public class RemoveIndexSettingsCommandIT extends ESIntegTestCase {
+
+    static final Setting<Integer> FOO = Setting.intSetting("index.foo", 1, Setting.Property.IndexScope, Setting.Property.Dynamic);
+    static final Setting<Integer> BAR = Setting.intSetting("index.bar", 2, Setting.Property.IndexScope, Setting.Property.Final);
+
+    public static class ExtraSettingsPlugin extends Plugin {
+        @Override
+        public List<Setting<?>> getSettings() {
+            return List.of(FOO, BAR);
+        }
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopy(super.nodePlugins(), ExtraSettingsPlugin.class);
+    }
+
+    public void testRemoveSettingsAbortedByUser() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        var node = internalCluster().startNode();
+        createIndex("test-index", Settings.builder().put(FOO.getKey(), 101).put(BAR.getKey(), 102).build());
+        ensureYellow("test-index");
+        Settings dataPathSettings = internalCluster().dataPathSettings(node);
+        ensureStableCluster(1);
+        internalCluster().stopRandomDataNode();
+
+        Settings nodeSettings = Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build();
+        ElasticsearchException error = expectThrows(
+            ElasticsearchException.class,
+            () -> removeIndexSettings(TestEnvironment.newEnvironment(nodeSettings), true, "index.foo")
+        );
+        assertThat(error.getMessage(), equalTo(ElasticsearchNodeCommand.ABORTED_BY_USER_MSG));
+        internalCluster().startNode(nodeSettings);
+    }
+
+    public void testRemoveSettingsSuccessful() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        var node = internalCluster().startNode();
+        Settings dataPathSettings = internalCluster().dataPathSettings(node);
+
+        int numIndices = randomIntBetween(1, 10);
+        int[] barValues = new int[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            String index = "test-index-" + i;
+            barValues[i] = between(1, 1000);
+            createIndex(index, Settings.builder().put(FOO.getKey(), between(1, 1000)).put(BAR.getKey(), barValues[i]).build());
+        }
+        int moreIndices = randomIntBetween(1, 10);
+        for (int i = 0; i < moreIndices; i++) {
+            createIndex("more-index-" + i, Settings.EMPTY);
+        }
+        internalCluster().stopNode(node);
+
+        Environment environment = TestEnvironment.newEnvironment(
+            Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build()
+        );
+
+        MockTerminal terminal = removeIndexSettings(environment, false, "index.foo");
+        assertThat(terminal.getOutput(), containsString(RemoveIndexSettingsCommand.SETTINGS_REMOVED_MSG));
+        for (int i = 0; i < numIndices; i++) {
+            assertThat(terminal.getOutput(), containsString("Index setting [index.foo] will be removed from index [[test-index-" + i));
+        }
+        for (int i = 0; i < moreIndices; i++) {
+            assertThat(terminal.getOutput(), not(containsString("Index setting [index.foo] will be removed from index [[more-index-" + i)));
+        }
+        Settings nodeSettings = Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build();
+        internalCluster().startNode(nodeSettings);
+
+        Map<String, Settings> getIndexSettings = client().admin().indices().prepareGetSettings("test-index-*").get().getIndexToSettings();
+        for (int i = 0; i < numIndices; i++) {
+            String index = "test-index-" + i;
+            Settings indexSettings = getIndexSettings.get(index);
+            assertFalse(indexSettings.hasValue("index.foo"));
+            assertThat(indexSettings.get("index.bar"), equalTo(Integer.toString(barValues[i])));
+        }
+        getIndexSettings = client().admin().indices().prepareGetSettings("more-index-*").get().getIndexToSettings();
+        for (int i = 0; i < moreIndices; i++) {
+            assertNotNull(getIndexSettings.get("more-index-" + i));
+        }
+    }
+
+    public void testSettingDoesNotMatch() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        var node = internalCluster().startNode();
+        createIndex("test-index", Settings.builder().put(FOO.getKey(), 101).put(BAR.getKey(), 102).build());
+        ensureYellow("test-index");
+        Settings dataPathSettings = internalCluster().dataPathSettings(node);
+        ensureStableCluster(1);
+        internalCluster().stopRandomDataNode();
+
+        Settings nodeSettings = Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build();
+        UserException error = expectThrows(
+            UserException.class,
+            () -> removeIndexSettings(TestEnvironment.newEnvironment(nodeSettings), true, "index.not_foo")
+        );
+        assertThat(error.getMessage(), containsString("No index setting matching [index.not_foo] were found on this node"));
+        internalCluster().startNode(nodeSettings);
+    }
+
+    private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, boolean abort, String... args)
+        throws Exception {
+        final MockTerminal terminal = MockTerminal.create();
+        final OptionSet options = command.getParser().parse(args);
+        final ProcessInfo processInfo = new ProcessInfo(Map.of(), Map.of(), createTempDir());
+        final String input;
+
+        if (abort) {
+            input = randomValueOtherThanMany(c -> c.equalsIgnoreCase("y"), () -> randomAlphaOfLength(1));
+        } else {
+            input = randomBoolean() ? "y" : "Y";
+        }
+
+        terminal.addTextInput(input);
+
+        try {
+            command.execute(terminal, options, environment, processInfo);
+        } finally {
+            assertThat(terminal.getOutput(), containsString(ElasticsearchNodeCommand.STOP_WARNING_MSG));
+        }
+
+        return terminal;
+    }
+
+    private MockTerminal removeIndexSettings(Environment environment, boolean abort, String... args) throws Exception {
+        final MockTerminal terminal = executeCommand(new RemoveIndexSettingsCommand(), environment, abort, args);
+        assertThat(terminal.getOutput(), containsString(RemoveIndexSettingsCommand.CONFIRMATION_MSG));
+        assertThat(terminal.getOutput(), containsString(RemoveIndexSettingsCommand.SETTINGS_REMOVED_MSG));
+        return terminal;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java
index 58f37ec220669..81044e8e3ad51 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java
@@ -20,6 +20,7 @@ class NodeToolCli extends MultiCommand {
         subcommands.put("detach-cluster", new DetachClusterCommand());
         subcommands.put("override-version", new OverrideNodeVersionCommand());
         subcommands.put("remove-settings", new RemoveSettingsCommand());
+        subcommands.put("remove-index-settings", new RemoveIndexSettingsCommand());
         subcommands.put("remove-customs", new RemoveCustomsCommand());
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommand.java
new file mode 100644
index 0000000000000..c6514f9cb4a0b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommand.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.gateway.PersistedClusterStateService;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+
+public class RemoveIndexSettingsCommand extends ElasticsearchNodeCommand {
+
+    static final String SETTINGS_REMOVED_MSG = "Index settings were successfully removed from the cluster state";
+    static final String CONFIRMATION_MSG = DELIMITER
+        + "\n"
+        + "You should only run this tool if you have incompatible index settings in the\n"
+        + "cluster state that prevent the cluster from forming.\n"
+        + "This tool can cause data loss and its use should be your last resort.\n"
+        + "\n"
+        + "Do you want to proceed?\n";
+
+    private final OptionSpec<String> arguments;
+
+    public RemoveIndexSettingsCommand() {
+        super("Removes index settings from the cluster state");
+        arguments = parser.nonOptions("index setting names");
+    }
+
+    @Override
+    protected void processDataPaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment env) throws IOException,
+        UserException {
+        final List<String> settingsToRemove = arguments.values(options);
+        if (settingsToRemove.isEmpty()) {
+            throw new UserException(ExitCodes.USAGE, "Must supply at least one index setting to remove");
+        }
+
+        final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths);
+
+        terminal.println(Terminal.Verbosity.VERBOSE, "Loading cluster state");
+        final Tuple<Long, ClusterState> termAndClusterState = loadTermAndClusterState(persistedClusterStateService, env);
+        final ClusterState oldClusterState = termAndClusterState.v2();
+        final Metadata.Builder newMetadataBuilder = Metadata.builder(oldClusterState.metadata());
+        int changes = 0;
+        for (IndexMetadata indexMetadata : oldClusterState.metadata()) {
+            Settings oldSettings = indexMetadata.getSettings();
+            Settings.Builder newSettings = Settings.builder().put(oldSettings);
+            boolean removed = false;
+            for (String settingToRemove : settingsToRemove) {
+                for (String settingKey : oldSettings.keySet()) {
+                    if (Regex.simpleMatch(settingToRemove, settingKey)) {
+                        terminal.println(
+                            "Index setting [" + settingKey + "] will be removed from index [" + indexMetadata.getIndex() + "]"
+                        );
+                        newSettings.remove(settingKey);
+                        removed = true;
+                    }
+                }
+            }
+            if (removed) {
+                newMetadataBuilder.put(IndexMetadata.builder(indexMetadata).settings(newSettings));
+                changes++;
+            }
+        }
+        if (changes == 0) {
+            throw new UserException(ExitCodes.USAGE, "No index setting matching " + settingsToRemove + " were found on this node");
+        }
+
+        final ClusterState newClusterState = ClusterState.builder(oldClusterState).metadata(newMetadataBuilder).build();
+        terminal.println(
+            Terminal.Verbosity.VERBOSE,
+            "[old cluster state = " + oldClusterState + ", new cluster state = " + newClusterState + "]"
+        );
+
+        confirm(terminal, CONFIRMATION_MSG);
+
+        try (PersistedClusterStateService.Writer writer = persistedClusterStateService.createWriter()) {
+            writer.writeFullStateAndCommit(termAndClusterState.v1(), newClusterState);
+        }
+
+        terminal.println(SETTINGS_REMOVED_MSG);
+    }
+}

From c374ad503668d68733476682030a4ba7472a7a65 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 3 Jun 2024 21:15:35 -0700
Subject: [PATCH 125/208] Fix esql enrich memory leak (#109275)

Block and Vector use a non-thread-safe RefCounted. Threads that increase
or decrease the references must have a happen-before relationship.
However, this order is not guaranteed in the enrich lookup for the
reference of selectedPositions. The driver can complete the
MergePositionsOperator, which decreases the reference count of
selectedPositions, while the finally block may also decrease it in a
separate thread. These actions occur without a defined happen-before
relationship.

Closes #108532
---
 .../esql/enrich/EnrichLookupService.java      | 166 ++++++++++--------
 1 file changed, 92 insertions(+), 74 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
index 69d988c958169..0ab57fe88d65b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.compute.data.BlockStreamInput;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.ElementType;
-import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.LocalCircuitBreaker;
 import org.elasticsearch.compute.data.OrdinalBytesRefBlock;
@@ -43,6 +42,7 @@
 import org.elasticsearch.compute.operator.OutputOperator;
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.RefCounted;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -247,33 +247,56 @@ private void doLookup(
         String matchField,
         Page inputPage,
         List<NamedExpression> extractFields,
-        ActionListener<Page> listener
+        final ActionListener<Page> listener
     ) {
         Block inputBlock = inputPage.getBlock(0);
-        final IntBlock selectedPositions;
-        final OrdinalBytesRefBlock ordinalsBytesRefBlock;
-        if (inputBlock instanceof BytesRefBlock bytesRefBlock && (ordinalsBytesRefBlock = bytesRefBlock.asOrdinals()) != null) {
-            inputBlock = ordinalsBytesRefBlock.getDictionaryVector().asBlock();
-            selectedPositions = ordinalsBytesRefBlock.getOrdinalsBlock();
-            selectedPositions.mustIncRef();
-        } else {
-            selectedPositions = IntVector.range(0, inputBlock.getPositionCount(), blockFactory).asBlock();
+        if (inputBlock.areAllValuesNull()) {
+            listener.onResponse(createNullResponse(inputPage.getPositionCount(), extractFields));
+            return;
         }
-        LocalCircuitBreaker localBreaker = null;
+        final List<Releasable> releasables = new ArrayList<>(6);
+        boolean started = false;
         try {
-            if (inputBlock.areAllValuesNull()) {
-                listener.onResponse(createNullResponse(inputPage.getPositionCount(), extractFields));
-                return;
-            }
-            ShardSearchRequest shardSearchRequest = new ShardSearchRequest(shardId, 0, AliasFilter.EMPTY);
-            SearchContext searchContext = searchService.createSearchContext(shardSearchRequest, SearchService.NO_TIMEOUT);
-            listener = ActionListener.runBefore(listener, searchContext::close);
-            localBreaker = new LocalCircuitBreaker(
+            final ShardSearchRequest shardSearchRequest = new ShardSearchRequest(shardId, 0, AliasFilter.EMPTY);
+            final SearchContext searchContext = searchService.createSearchContext(shardSearchRequest, SearchService.NO_TIMEOUT);
+            releasables.add(searchContext);
+            final LocalCircuitBreaker localBreaker = new LocalCircuitBreaker(
                 blockFactory.breaker(),
                 localBreakerSettings.overReservedBytes(),
                 localBreakerSettings.maxOverReservedBytes()
             );
-            DriverContext driverContext = new DriverContext(bigArrays, blockFactory.newChildFactory(localBreaker));
+            releasables.add(localBreaker);
+            final DriverContext driverContext = new DriverContext(bigArrays, blockFactory.newChildFactory(localBreaker));
+            final ElementType[] mergingTypes = new ElementType[extractFields.size()];
+            for (int i = 0; i < extractFields.size(); i++) {
+                mergingTypes[i] = PlannerUtils.toElementType(extractFields.get(i).dataType());
+            }
+            final int[] mergingChannels = IntStream.range(0, extractFields.size()).map(i -> i + 2).toArray();
+            final MergePositionsOperator mergePositionsOperator;
+            final OrdinalBytesRefBlock ordinalsBytesRefBlock;
+            if (inputBlock instanceof BytesRefBlock bytesRefBlock && (ordinalsBytesRefBlock = bytesRefBlock.asOrdinals()) != null) {
+                inputBlock = ordinalsBytesRefBlock.getDictionaryVector().asBlock();
+                var selectedPositions = ordinalsBytesRefBlock.getOrdinalsBlock();
+                mergePositionsOperator = new MergePositionsOperator(
+                    1,
+                    mergingChannels,
+                    mergingTypes,
+                    selectedPositions,
+                    driverContext.blockFactory()
+                );
+
+            } else {
+                try (var selectedPositions = IntVector.range(0, inputBlock.getPositionCount(), blockFactory).asBlock()) {
+                    mergePositionsOperator = new MergePositionsOperator(
+                        1,
+                        mergingChannels,
+                        mergingTypes,
+                        selectedPositions,
+                        driverContext.blockFactory()
+                    );
+                }
+            }
+            releasables.add(mergePositionsOperator);
             SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext();
             MappedFieldType fieldType = searchExecutionContext.getFieldType(matchField);
             var queryList = switch (matchType) {
@@ -287,57 +310,13 @@ private void doLookup(
                 queryList,
                 searchExecutionContext.getIndexReader()
             );
-            List<Operator> intermediateOperators = new ArrayList<>(extractFields.size() + 2);
-            final ElementType[] mergingTypes = new ElementType[extractFields.size()];
-            // load the fields
-            List<ValuesSourceReaderOperator.FieldInfo> fields = new ArrayList<>(extractFields.size());
-            for (int i = 0; i < extractFields.size(); i++) {
-                NamedExpression extractField = extractFields.get(i);
-                final ElementType elementType = PlannerUtils.toElementType(extractField.dataType());
-                mergingTypes[i] = elementType;
-                EsPhysicalOperationProviders.ShardContext ctx = new EsPhysicalOperationProviders.DefaultShardContext(
-                    0,
-                    searchContext.getSearchExecutionContext(),
-                    searchContext.request().getAliasFilter()
-                );
-                BlockLoader loader = ctx.blockLoader(
-                    extractField instanceof Alias a ? ((NamedExpression) a.child()).name() : extractField.name(),
-                    EsqlDataTypes.isUnsupported(extractField.dataType()),
-                    MappedFieldType.FieldExtractPreference.NONE
-                );
-                fields.add(
-                    new ValuesSourceReaderOperator.FieldInfo(
-                        extractField.name(),
-                        PlannerUtils.toElementType(extractField.dataType()),
-                        shardIdx -> {
-                            if (shardIdx != 0) {
-                                throw new IllegalStateException("only one shard");
-                            }
-                            return loader;
-                        }
-                    )
-                );
-            }
-            intermediateOperators.add(
-                new ValuesSourceReaderOperator(
-                    driverContext.blockFactory(),
-                    fields,
-                    List.of(
-                        new ValuesSourceReaderOperator.ShardContext(
-                            searchContext.searcher().getIndexReader(),
-                            searchContext::newSourceLoader
-                        )
-                    ),
-                    0
-                )
-            );
-            // merging field-values by position
-            final int[] mergingChannels = IntStream.range(0, extractFields.size()).map(i -> i + 2).toArray();
-            intermediateOperators.add(
-                new MergePositionsOperator(1, mergingChannels, mergingTypes, selectedPositions, driverContext.blockFactory())
-            );
+            releasables.add(queryOperator);
+            var extractFieldsOperator = extractFieldsOperator(searchContext, driverContext, extractFields);
+            releasables.add(extractFieldsOperator);
+
             AtomicReference<Page> result = new AtomicReference<>();
             OutputOperator outputOperator = new OutputOperator(List.of(), Function.identity(), result::set);
+            releasables.add(outputOperator);
             Driver driver = new Driver(
                 "enrich-lookup:" + sessionId,
                 System.currentTimeMillis(),
@@ -353,18 +332,16 @@ private void doLookup(
                     inputPage.getPositionCount()
                 ),
                 queryOperator,
-                intermediateOperators,
+                List.of(extractFieldsOperator, mergePositionsOperator),
                 outputOperator,
                 Driver.DEFAULT_STATUS_INTERVAL,
-                localBreaker
+                Releasables.wrap(searchContext, localBreaker)
             );
             task.addListener(() -> {
                 String reason = Objects.requireNonNullElse(task.getReasonCancelled(), "task was cancelled");
                 driver.cancel(reason);
             });
-
             var threadContext = transportService.getThreadPool().getThreadContext();
-            localBreaker = null;
             Driver.start(threadContext, executor, driver, Driver.DEFAULT_MAX_ITERATIONS, listener.map(ignored -> {
                 Page out = result.get();
                 if (out == null) {
@@ -372,11 +349,52 @@ private void doLookup(
                 }
                 return out;
             }));
+            started = true;
         } catch (Exception e) {
             listener.onFailure(e);
         } finally {
-            Releasables.close(selectedPositions, localBreaker);
+            if (started == false) {
+                Releasables.close(releasables);
+            }
+        }
+    }
+
+    private static Operator extractFieldsOperator(
+        SearchContext searchContext,
+        DriverContext driverContext,
+        List<NamedExpression> extractFields
+    ) {
+        EsPhysicalOperationProviders.ShardContext shardContext = new EsPhysicalOperationProviders.DefaultShardContext(
+            0,
+            searchContext.getSearchExecutionContext(),
+            searchContext.request().getAliasFilter()
+        );
+        List<ValuesSourceReaderOperator.FieldInfo> fields = new ArrayList<>(extractFields.size());
+        for (NamedExpression extractField : extractFields) {
+            BlockLoader loader = shardContext.blockLoader(
+                extractField instanceof Alias a ? ((NamedExpression) a.child()).name() : extractField.name(),
+                EsqlDataTypes.isUnsupported(extractField.dataType()),
+                MappedFieldType.FieldExtractPreference.NONE
+            );
+            fields.add(
+                new ValuesSourceReaderOperator.FieldInfo(
+                    extractField.name(),
+                    PlannerUtils.toElementType(extractField.dataType()),
+                    shardIdx -> {
+                        if (shardIdx != 0) {
+                            throw new IllegalStateException("only one shard");
+                        }
+                        return loader;
+                    }
+                )
+            );
         }
+        return new ValuesSourceReaderOperator(
+            driverContext.blockFactory(),
+            fields,
+            List.of(new ValuesSourceReaderOperator.ShardContext(searchContext.searcher().getIndexReader(), searchContext::newSourceLoader)),
+            0
+        );
     }
 
     private Page createNullResponse(int positionCount, List<NamedExpression> extractFields) {

From 191cacae4855eed6eb172bb29fe903ba83e924d7 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Mon, 3 Jun 2024 23:09:26 -0700
Subject: [PATCH 126/208] Revert "Fix esql enrich memory leak (#109275)"

This reverts commit c374ad503668d68733476682030a4ba7472a7a65.
---
 .../esql/enrich/EnrichLookupService.java      | 166 ++++++++----------
 1 file changed, 74 insertions(+), 92 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
index 0ab57fe88d65b..69d988c958169 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.compute.data.BlockStreamInput;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.ElementType;
+import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.LocalCircuitBreaker;
 import org.elasticsearch.compute.data.OrdinalBytesRefBlock;
@@ -42,7 +43,6 @@
 import org.elasticsearch.compute.operator.OutputOperator;
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.RefCounted;
-import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -247,56 +247,33 @@ private void doLookup(
         String matchField,
         Page inputPage,
         List<NamedExpression> extractFields,
-        final ActionListener<Page> listener
+        ActionListener<Page> listener
     ) {
         Block inputBlock = inputPage.getBlock(0);
-        if (inputBlock.areAllValuesNull()) {
-            listener.onResponse(createNullResponse(inputPage.getPositionCount(), extractFields));
-            return;
+        final IntBlock selectedPositions;
+        final OrdinalBytesRefBlock ordinalsBytesRefBlock;
+        if (inputBlock instanceof BytesRefBlock bytesRefBlock && (ordinalsBytesRefBlock = bytesRefBlock.asOrdinals()) != null) {
+            inputBlock = ordinalsBytesRefBlock.getDictionaryVector().asBlock();
+            selectedPositions = ordinalsBytesRefBlock.getOrdinalsBlock();
+            selectedPositions.mustIncRef();
+        } else {
+            selectedPositions = IntVector.range(0, inputBlock.getPositionCount(), blockFactory).asBlock();
         }
-        final List<Releasable> releasables = new ArrayList<>(6);
-        boolean started = false;
+        LocalCircuitBreaker localBreaker = null;
         try {
-            final ShardSearchRequest shardSearchRequest = new ShardSearchRequest(shardId, 0, AliasFilter.EMPTY);
-            final SearchContext searchContext = searchService.createSearchContext(shardSearchRequest, SearchService.NO_TIMEOUT);
-            releasables.add(searchContext);
-            final LocalCircuitBreaker localBreaker = new LocalCircuitBreaker(
+            if (inputBlock.areAllValuesNull()) {
+                listener.onResponse(createNullResponse(inputPage.getPositionCount(), extractFields));
+                return;
+            }
+            ShardSearchRequest shardSearchRequest = new ShardSearchRequest(shardId, 0, AliasFilter.EMPTY);
+            SearchContext searchContext = searchService.createSearchContext(shardSearchRequest, SearchService.NO_TIMEOUT);
+            listener = ActionListener.runBefore(listener, searchContext::close);
+            localBreaker = new LocalCircuitBreaker(
                 blockFactory.breaker(),
                 localBreakerSettings.overReservedBytes(),
                 localBreakerSettings.maxOverReservedBytes()
             );
-            releasables.add(localBreaker);
-            final DriverContext driverContext = new DriverContext(bigArrays, blockFactory.newChildFactory(localBreaker));
-            final ElementType[] mergingTypes = new ElementType[extractFields.size()];
-            for (int i = 0; i < extractFields.size(); i++) {
-                mergingTypes[i] = PlannerUtils.toElementType(extractFields.get(i).dataType());
-            }
-            final int[] mergingChannels = IntStream.range(0, extractFields.size()).map(i -> i + 2).toArray();
-            final MergePositionsOperator mergePositionsOperator;
-            final OrdinalBytesRefBlock ordinalsBytesRefBlock;
-            if (inputBlock instanceof BytesRefBlock bytesRefBlock && (ordinalsBytesRefBlock = bytesRefBlock.asOrdinals()) != null) {
-                inputBlock = ordinalsBytesRefBlock.getDictionaryVector().asBlock();
-                var selectedPositions = ordinalsBytesRefBlock.getOrdinalsBlock();
-                mergePositionsOperator = new MergePositionsOperator(
-                    1,
-                    mergingChannels,
-                    mergingTypes,
-                    selectedPositions,
-                    driverContext.blockFactory()
-                );
-
-            } else {
-                try (var selectedPositions = IntVector.range(0, inputBlock.getPositionCount(), blockFactory).asBlock()) {
-                    mergePositionsOperator = new MergePositionsOperator(
-                        1,
-                        mergingChannels,
-                        mergingTypes,
-                        selectedPositions,
-                        driverContext.blockFactory()
-                    );
-                }
-            }
-            releasables.add(mergePositionsOperator);
+            DriverContext driverContext = new DriverContext(bigArrays, blockFactory.newChildFactory(localBreaker));
             SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext();
             MappedFieldType fieldType = searchExecutionContext.getFieldType(matchField);
             var queryList = switch (matchType) {
@@ -310,13 +287,57 @@ private void doLookup(
                 queryList,
                 searchExecutionContext.getIndexReader()
             );
-            releasables.add(queryOperator);
-            var extractFieldsOperator = extractFieldsOperator(searchContext, driverContext, extractFields);
-            releasables.add(extractFieldsOperator);
-
+            List<Operator> intermediateOperators = new ArrayList<>(extractFields.size() + 2);
+            final ElementType[] mergingTypes = new ElementType[extractFields.size()];
+            // load the fields
+            List<ValuesSourceReaderOperator.FieldInfo> fields = new ArrayList<>(extractFields.size());
+            for (int i = 0; i < extractFields.size(); i++) {
+                NamedExpression extractField = extractFields.get(i);
+                final ElementType elementType = PlannerUtils.toElementType(extractField.dataType());
+                mergingTypes[i] = elementType;
+                EsPhysicalOperationProviders.ShardContext ctx = new EsPhysicalOperationProviders.DefaultShardContext(
+                    0,
+                    searchContext.getSearchExecutionContext(),
+                    searchContext.request().getAliasFilter()
+                );
+                BlockLoader loader = ctx.blockLoader(
+                    extractField instanceof Alias a ? ((NamedExpression) a.child()).name() : extractField.name(),
+                    EsqlDataTypes.isUnsupported(extractField.dataType()),
+                    MappedFieldType.FieldExtractPreference.NONE
+                );
+                fields.add(
+                    new ValuesSourceReaderOperator.FieldInfo(
+                        extractField.name(),
+                        PlannerUtils.toElementType(extractField.dataType()),
+                        shardIdx -> {
+                            if (shardIdx != 0) {
+                                throw new IllegalStateException("only one shard");
+                            }
+                            return loader;
+                        }
+                    )
+                );
+            }
+            intermediateOperators.add(
+                new ValuesSourceReaderOperator(
+                    driverContext.blockFactory(),
+                    fields,
+                    List.of(
+                        new ValuesSourceReaderOperator.ShardContext(
+                            searchContext.searcher().getIndexReader(),
+                            searchContext::newSourceLoader
+                        )
+                    ),
+                    0
+                )
+            );
+            // merging field-values by position
+            final int[] mergingChannels = IntStream.range(0, extractFields.size()).map(i -> i + 2).toArray();
+            intermediateOperators.add(
+                new MergePositionsOperator(1, mergingChannels, mergingTypes, selectedPositions, driverContext.blockFactory())
+            );
             AtomicReference<Page> result = new AtomicReference<>();
             OutputOperator outputOperator = new OutputOperator(List.of(), Function.identity(), result::set);
-            releasables.add(outputOperator);
             Driver driver = new Driver(
                 "enrich-lookup:" + sessionId,
                 System.currentTimeMillis(),
@@ -332,16 +353,18 @@ private void doLookup(
                     inputPage.getPositionCount()
                 ),
                 queryOperator,
-                List.of(extractFieldsOperator, mergePositionsOperator),
+                intermediateOperators,
                 outputOperator,
                 Driver.DEFAULT_STATUS_INTERVAL,
-                Releasables.wrap(searchContext, localBreaker)
+                localBreaker
             );
             task.addListener(() -> {
                 String reason = Objects.requireNonNullElse(task.getReasonCancelled(), "task was cancelled");
                 driver.cancel(reason);
             });
+
             var threadContext = transportService.getThreadPool().getThreadContext();
+            localBreaker = null;
             Driver.start(threadContext, executor, driver, Driver.DEFAULT_MAX_ITERATIONS, listener.map(ignored -> {
                 Page out = result.get();
                 if (out == null) {
@@ -349,52 +372,11 @@ private void doLookup(
                 }
                 return out;
             }));
-            started = true;
         } catch (Exception e) {
             listener.onFailure(e);
         } finally {
-            if (started == false) {
-                Releasables.close(releasables);
-            }
-        }
-    }
-
-    private static Operator extractFieldsOperator(
-        SearchContext searchContext,
-        DriverContext driverContext,
-        List<NamedExpression> extractFields
-    ) {
-        EsPhysicalOperationProviders.ShardContext shardContext = new EsPhysicalOperationProviders.DefaultShardContext(
-            0,
-            searchContext.getSearchExecutionContext(),
-            searchContext.request().getAliasFilter()
-        );
-        List<ValuesSourceReaderOperator.FieldInfo> fields = new ArrayList<>(extractFields.size());
-        for (NamedExpression extractField : extractFields) {
-            BlockLoader loader = shardContext.blockLoader(
-                extractField instanceof Alias a ? ((NamedExpression) a.child()).name() : extractField.name(),
-                EsqlDataTypes.isUnsupported(extractField.dataType()),
-                MappedFieldType.FieldExtractPreference.NONE
-            );
-            fields.add(
-                new ValuesSourceReaderOperator.FieldInfo(
-                    extractField.name(),
-                    PlannerUtils.toElementType(extractField.dataType()),
-                    shardIdx -> {
-                        if (shardIdx != 0) {
-                            throw new IllegalStateException("only one shard");
-                        }
-                        return loader;
-                    }
-                )
-            );
+            Releasables.close(selectedPositions, localBreaker);
         }
-        return new ValuesSourceReaderOperator(
-            driverContext.blockFactory(),
-            fields,
-            List.of(new ValuesSourceReaderOperator.ShardContext(searchContext.searcher().getIndexReader(), searchContext::newSourceLoader)),
-            0
-        );
     }
 
     private Page createNullResponse(int positionCount, List<NamedExpression> extractFields) {

From d543d91f0257faef0df36c03cf5ab5b8c5c4ca38 Mon Sep 17 00:00:00 2001
From: Jedr Blaszyk <jedrazb@gmail.com>
Date: Tue, 4 Jun 2024 11:58:37 +0200
Subject: [PATCH 127/208] [Connector API] Implement _features endpoint
 (#109248)

---
 .../api/connector.update_features.json        |  38 +++++
 .../170_connector_update_features.yml         | 108 ++++++++++++++
 .../xpack/application/EnterpriseSearch.java   |   5 +
 .../application/connector/Connector.java      |   2 +-
 .../connector/ConnectorFeatures.java          |  16 +++
 .../connector/ConnectorIndexService.java      |  27 ++++
 .../RestUpdateConnectorFeaturesAction.java    |  48 +++++++
 ...ransportUpdateConnectorFeaturesAction.java |  50 +++++++
 .../action/UpdateConnectorFeaturesAction.java | 133 ++++++++++++++++++
 .../connector/ConnectorIndexServiceTests.java |  73 ++++++++++
 .../connector/ConnectorTestUtils.java         |   2 +-
 ...turesActionRequestBWCSerializingTests.java |  51 +++++++
 .../xpack/security/operator/Constants.java    |   1 +
 13 files changed, 552 insertions(+), 2 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_features.json
 create mode 100644 x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml
 create mode 100644 x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFeaturesAction.java
 create mode 100644 x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java
 create mode 100644 x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java
 create mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_features.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_features.json
new file mode 100644
index 0000000000000..b488e19262c2e
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.update_features.json
@@ -0,0 +1,38 @@
+{
+  "connector.update_features": {
+    "documentation": {
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/update-connector-features-api.html",
+      "description": "Updates the connector features in the connector document."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_connector/{connector_id}/_features",
+          "methods": [
+            "PUT"
+          ],
+          "parts": {
+            "connector_id": {
+              "type": "string",
+              "description": "The unique identifier of the connector to be updated."
+            }
+          }
+        }
+      ]
+    },
+    "body": {
+      "description": "An object containing the connector's features definition.",
+      "required": true
+    }
+  }
+}
diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml
new file mode 100644
index 0000000000000..0964e4f50ebde
--- /dev/null
+++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/170_connector_update_features.yml
@@ -0,0 +1,108 @@
+setup:
+  - requires:
+      cluster_features: ["gte_v8.15.0"]
+      reason: Introduced in 8.15.0
+
+  - do:
+      connector.put:
+        connector_id: test-connector
+        body:
+          index_name: search-1-test
+          name: my-connector
+          language: pl
+          is_native: false
+          service_type: super-connector
+
+---
+"Update Connector Features":
+  - do:
+      connector.update_features:
+        connector_id: test-connector
+        body:
+          features:
+            document_level_security: { enabled: true }
+            native_connector_api_keys: { enabled: true }
+            incremental_sync: { enabled: false }
+            sync_rules:
+              basic: { enabled: true }
+              advanced: { enabled: false }
+
+
+  - match: { result: updated }
+
+  - do:
+      connector.get:
+        connector_id: test-connector
+
+  - match: { features.document_level_security.enabled: true }
+  - match: { features.native_connector_api_keys.enabled: true }
+  - match: { features.incremental_sync.enabled: false }
+  - match: { features.sync_rules.basic.enabled: true }
+  - match: { features.sync_rules.advanced.enabled: false }
+
+---
+"Update Connector Features - Partial Update":
+  - do:
+      connector.update_features:
+        connector_id: test-connector
+        body:
+          features:
+            document_level_security: { enabled: true }
+
+
+  - match: { result: updated }
+
+  - do:
+      connector.get:
+        connector_id: test-connector
+
+  - match: { features.document_level_security.enabled: true }
+
+
+  - do:
+      connector.update_features:
+        connector_id: test-connector
+        body:
+          features:
+            native_connector_api_keys: { enabled: true }
+
+
+  - match: { result: updated }
+
+  - do:
+      connector.get:
+        connector_id: test-connector
+
+  # Assert that existing feature remains unchanged
+  - match: { features.document_level_security.enabled: true }
+  - match: { features.native_connector_api_keys.enabled: true }
+
+---
+"Update Connector Features - 404 when connector doesn't exist":
+  - do:
+      catch: "missing"
+      connector.update_features:
+        connector_id: test-non-existent-connector
+        body:
+          features:
+            native_connector_api_keys: { enabled: true }
+
+---
+"Update Connector Features - 400 status code when connector_id is empty":
+  - do:
+      catch: "bad_request"
+      connector.update_features:
+        connector_id: ""
+        body:
+          features:
+            native_connector_api_keys: { enabled: true }
+
+---
+"Update Connector Features - 400 status code when payload unknown":
+  - do:
+      catch: "bad_request"
+      connector.update_features:
+        connector_id: test-connector
+        body:
+          featuresss:
+            not_a_feature: 12423
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java
index bc3da1a82fba4..871bf7fb122b9 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java
@@ -58,6 +58,7 @@
 import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorApiKeyIdAction;
 import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorConfigurationAction;
 import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorErrorAction;
+import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorFeaturesAction;
 import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorFilteringAction;
 import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorFilteringValidationAction;
 import org.elasticsearch.xpack.application.connector.action.RestUpdateConnectorIndexNameAction;
@@ -78,6 +79,7 @@
 import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorApiKeyIdAction;
 import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorConfigurationAction;
 import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorErrorAction;
+import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorFeaturesAction;
 import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorFilteringAction;
 import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorFilteringValidationAction;
 import org.elasticsearch.xpack.application.connector.action.TransportUpdateConnectorIndexNameAction;
@@ -93,6 +95,7 @@
 import org.elasticsearch.xpack.application.connector.action.UpdateConnectorApiKeyIdAction;
 import org.elasticsearch.xpack.application.connector.action.UpdateConnectorConfigurationAction;
 import org.elasticsearch.xpack.application.connector.action.UpdateConnectorErrorAction;
+import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFeaturesAction;
 import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFilteringAction;
 import org.elasticsearch.xpack.application.connector.action.UpdateConnectorFilteringValidationAction;
 import org.elasticsearch.xpack.application.connector.action.UpdateConnectorIndexNameAction;
@@ -267,6 +270,7 @@ protected XPackLicenseState getLicenseState() {
                     new ActionHandler<>(UpdateConnectorApiKeyIdAction.INSTANCE, TransportUpdateConnectorApiKeyIdAction.class),
                     new ActionHandler<>(UpdateConnectorConfigurationAction.INSTANCE, TransportUpdateConnectorConfigurationAction.class),
                     new ActionHandler<>(UpdateConnectorErrorAction.INSTANCE, TransportUpdateConnectorErrorAction.class),
+                    new ActionHandler<>(UpdateConnectorFeaturesAction.INSTANCE, TransportUpdateConnectorFeaturesAction.class),
                     new ActionHandler<>(UpdateConnectorFilteringAction.INSTANCE, TransportUpdateConnectorFilteringAction.class),
                     new ActionHandler<>(UpdateConnectorActiveFilteringAction.INSTANCE, TransportUpdateConnectorActiveFilteringAction.class),
                     new ActionHandler<>(
@@ -368,6 +372,7 @@ public List<RestHandler> getRestHandlers(
                     new RestUpdateConnectorConfigurationAction(),
                     new RestUpdateConnectorErrorAction(),
                     new RestUpdateConnectorActiveFilteringAction(),
+                    new RestUpdateConnectorFeaturesAction(),
                     new RestUpdateConnectorFilteringValidationAction(),
                     new RestUpdateConnectorFilteringAction(),
                     new RestUpdateConnectorIndexNameAction(),
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java
index e9447149c7e6c..62f42d9a16ead 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java
@@ -213,7 +213,7 @@ public Connector(StreamInput in) throws IOException {
     static final ParseField CUSTOM_SCHEDULING_FIELD = new ParseField("custom_scheduling");
     public static final ParseField DESCRIPTION_FIELD = new ParseField("description");
     public static final ParseField ERROR_FIELD = new ParseField("error");
-    static final ParseField FEATURES_FIELD = new ParseField("features");
+    public static final ParseField FEATURES_FIELD = new ParseField("features");
     public static final ParseField FILTERING_FIELD = new ParseField("filtering");
     public static final ParseField INDEX_NAME_FIELD = new ParseField("index_name");
     public static final ParseField IS_NATIVE_FIELD = new ParseField("is_native");
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java
index 1b2e7209e41e5..0b9a72f06ad53 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java
@@ -102,6 +102,22 @@ public static ConnectorFeatures fromXContentBytes(BytesReference source, XConten
         }
     }
 
+    public FeatureEnabled getDocumentLevelSecurityEnabled() {
+        return documentLevelSecurityEnabled;
+    }
+
+    public FeatureEnabled getIncrementalSyncEnabled() {
+        return incrementalSyncEnabled;
+    }
+
+    public FeatureEnabled getNativeConnectorAPIKeysEnabled() {
+        return nativeConnectorAPIKeysEnabled;
+    }
+
+    public SyncRulesFeatures getSyncRulesFeatures() {
+        return syncRulesFeatures;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java
index 50e2633bb8c76..e5314a20bdccf 100644
--- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java
@@ -549,6 +549,33 @@ public void updateConnectorFiltering(String connectorId, List<ConnectorFiltering
         }
     }
 
+    /**
+     * Updates the features of a given {@link Connector}.
+     *
+     * @param connectorId The ID of the {@link Connector} to be updated.
+     * @param features    An instance of {@link ConnectorFeatures}
+     * @param listener    Listener to respond to a successful response or an error.
+     */
+    public void updateConnectorFeatures(String connectorId, ConnectorFeatures features, ActionListener<UpdateResponse> listener) {
+        try {
+            final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc(
+                new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
+                    .id(connectorId)
+                    .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                    .source(Map.of(Connector.FEATURES_FIELD.getPreferredName(), features))
+            );
+            client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
+                if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+                    l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+                    return;
+                }
+                l.onResponse(updateResponse);
+            }));
+        } catch (Exception e) {
+            listener.onFailure(e);
+        }
+    }
+
     /**
      * Updates the draft filtering in a given {@link Connector}.
      *
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFeaturesAction.java
new file mode 100644
index 0000000000000..48bf87b114548
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestUpdateConnectorFeaturesAction.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.connector.action;
+
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.Scope;
+import org.elasticsearch.rest.ServerlessScope;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.application.EnterpriseSearch;
+
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.PUT;
+
+@ServerlessScope(Scope.PUBLIC)
+public class RestUpdateConnectorFeaturesAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "connector_update_features_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(PUT, "/" + EnterpriseSearch.CONNECTOR_API_ENDPOINT + "/{connector_id}/_features"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+        UpdateConnectorFeaturesAction.Request request = UpdateConnectorFeaturesAction.Request.fromXContentBytes(
+            restRequest.param("connector_id"),
+            restRequest.content(),
+            restRequest.getXContentType()
+        );
+        return channel -> client.execute(
+            UpdateConnectorFeaturesAction.INSTANCE,
+            request,
+            new RestToXContentListener<>(channel, ConnectorUpdateActionResponse::status)
+        );
+    }
+}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java
new file mode 100644
index 0000000000000..c86ddf902519f
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportUpdateConnectorFeaturesAction.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.connector.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
+
+public class TransportUpdateConnectorFeaturesAction extends HandledTransportAction<
+    UpdateConnectorFeaturesAction.Request,
+    ConnectorUpdateActionResponse> {
+
+    protected final ConnectorIndexService connectorIndexService;
+
+    @Inject
+    public TransportUpdateConnectorFeaturesAction(TransportService transportService, ActionFilters actionFilters, Client client) {
+        super(
+            UpdateConnectorFeaturesAction.NAME,
+            transportService,
+            actionFilters,
+            UpdateConnectorFeaturesAction.Request::new,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE
+        );
+        this.connectorIndexService = new ConnectorIndexService(client);
+    }
+
+    @Override
+    protected void doExecute(
+        Task task,
+        UpdateConnectorFeaturesAction.Request request,
+        ActionListener<ConnectorUpdateActionResponse> listener
+    ) {
+        connectorIndexService.updateConnectorFeatures(
+            request.getConnectorId(),
+            request.getFeatures(),
+            listener.map(r -> new ConnectorUpdateActionResponse(r.getResult()))
+        );
+    }
+}
diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java
new file mode 100644
index 0000000000000..c1f62c0efe6e8
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.connector.action;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.application.connector.Connector;
+import org.elasticsearch.xpack.application.connector.ConnectorFeatures;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class UpdateConnectorFeaturesAction {
+
+    public static final String NAME = "indices:data/write/xpack/connector/update_features";
+    public static final ActionType<ConnectorUpdateActionResponse> INSTANCE = new ActionType<>(NAME);
+
+    private UpdateConnectorFeaturesAction() {/* no instances */}
+
+    public static class Request extends ConnectorActionRequest implements ToXContentObject {
+
+        private final String connectorId;
+
+        private final ConnectorFeatures features;
+
+        public Request(String connectorId, ConnectorFeatures features) {
+            this.connectorId = connectorId;
+            this.features = features;
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.connectorId = in.readString();
+            this.features = in.readOptionalWriteable(ConnectorFeatures::new);
+        }
+
+        public String getConnectorId() {
+            return connectorId;
+        }
+
+        public ConnectorFeatures getFeatures() {
+            return features;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            ActionRequestValidationException validationException = null;
+
+            if (Strings.isNullOrEmpty(connectorId)) {
+                validationException = addValidationError("[connector_id] cannot be [null] or [\"\"].", validationException);
+            }
+
+            return validationException;
+        }
+
+        private static final ConstructingObjectParser<Request, String> PARSER = new ConstructingObjectParser<>(
+            "connector_update_features_request",
+            false,
+            ((args, connectorId) -> new UpdateConnectorFeaturesAction.Request(connectorId, (ConnectorFeatures) args[0]))
+        );
+
+        static {
+            PARSER.declareObject(optionalConstructorArg(), (p, c) -> ConnectorFeatures.fromXContent(p), Connector.FEATURES_FIELD);
+        }
+
+        public static UpdateConnectorFeaturesAction.Request fromXContentBytes(
+            String connectorId,
+            BytesReference source,
+            XContentType xContentType
+        ) {
+            try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) {
+                return UpdateConnectorFeaturesAction.Request.fromXContent(parser, connectorId);
+            } catch (IOException e) {
+                throw new ElasticsearchParseException("Failed to parse: " + source.utf8ToString(), e);
+            }
+        }
+
+        public static UpdateConnectorFeaturesAction.Request fromXContent(XContentParser parser, String connectorId) throws IOException {
+            return PARSER.parse(parser, connectorId);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            {
+                builder.field(Connector.FEATURES_FIELD.getPreferredName(), features);
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(connectorId);
+            out.writeOptionalWriteable(features);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request request = (Request) o;
+            return Objects.equals(connectorId, request.connectorId) && Objects.equals(features, request.features);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(connectorId, features);
+        }
+
+    }
+}
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java
index 045cb725e477c..21a0fede4675e 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java
@@ -56,7 +56,9 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomConnectorFeatures;
 import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.getRandomCronExpression;
+import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.randomConnectorFeatureEnabled;
 import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.registerSimplifiedConnectorIndexTemplates;
 import static org.hamcrest.CoreMatchers.anyOf;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -240,6 +242,51 @@ public void testUpdateConnectorPipeline() throws Exception {
         assertThat(updatedPipeline, equalTo(indexedConnector.getPipeline()));
     }
 
+    public void testUpdateConnectorFeatures() throws Exception {
+        Connector connector = ConnectorTestUtils.getRandomConnector();
+        String connectorId = randomUUID();
+
+        ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector);
+        assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK)));
+
+        ConnectorFeatures newFeatures = getRandomConnectorFeatures();
+
+        DocWriteResponse updateResponse = awaitUpdateConnectorFeatures(connectorId, newFeatures);
+        assertThat(updateResponse.status(), equalTo(RestStatus.OK));
+        Connector indexedConnector = awaitGetConnector(connectorId);
+        assertThat(newFeatures, equalTo(indexedConnector.getFeatures()));
+
+    }
+
+    public void testUpdateConnectorFeatures_partialUpdate() throws Exception {
+        Connector connector = ConnectorTestUtils.getRandomConnector();
+        String connectorId = randomUUID();
+
+        ConnectorCreateActionResponse resp = awaitCreateConnector(connectorId, connector);
+        assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK)));
+
+        ConnectorFeatures features = getRandomConnectorFeatures();
+
+        awaitUpdateConnectorFeatures(connectorId, features);
+
+        Connector indexedConnector = awaitGetConnector(connectorId);
+        assertThat(features, equalTo(indexedConnector.getFeatures()));
+
+        // Partial update of DLS feature
+        ConnectorFeatures dlsFeature = new ConnectorFeatures.Builder().setDocumentLevelSecurityEnabled(randomConnectorFeatureEnabled())
+            .build();
+        awaitUpdateConnectorFeatures(connectorId, dlsFeature);
+        indexedConnector = awaitGetConnector(connectorId);
+
+        // Assert that partial update was applied
+        assertThat(dlsFeature.getDocumentLevelSecurityEnabled(), equalTo(indexedConnector.getFeatures().getDocumentLevelSecurityEnabled()));
+
+        // Assert other features are unchanged
+        assertThat(features.getSyncRulesFeatures(), equalTo(indexedConnector.getFeatures().getSyncRulesFeatures()));
+        assertThat(features.getNativeConnectorAPIKeysEnabled(), equalTo(indexedConnector.getFeatures().getNativeConnectorAPIKeysEnabled()));
+        assertThat(features.getIncrementalSyncEnabled(), equalTo(indexedConnector.getFeatures().getIncrementalSyncEnabled()));
+    }
+
     public void testUpdateConnectorFiltering() throws Exception {
         Connector connector = ConnectorTestUtils.getRandomConnector();
         String connectorId = randomUUID();
@@ -890,6 +937,32 @@ public void onFailure(Exception e) {
         return resp.get();
     }
 
+    private UpdateResponse awaitUpdateConnectorFeatures(String connectorId, ConnectorFeatures features) throws Exception {
+        CountDownLatch latch = new CountDownLatch(1);
+        final AtomicReference<UpdateResponse> resp = new AtomicReference<>(null);
+        final AtomicReference<Exception> exc = new AtomicReference<>(null);
+        connectorIndexService.updateConnectorFeatures(connectorId, features, new ActionListener<>() {
+            @Override
+            public void onResponse(UpdateResponse indexResponse) {
+                resp.set(indexResponse);
+                latch.countDown();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exc.set(e);
+                latch.countDown();
+            }
+        });
+
+        assertTrue("Timeout waiting for update features request", latch.await(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+        if (exc.get() != null) {
+            throw exc.get();
+        }
+        assertNotNull("Received null response from update features request", resp.get());
+        return resp.get();
+    }
+
     private UpdateResponse awaitUpdateConnectorFiltering(String connectorId, List<ConnectorFiltering> filtering) throws Exception {
         CountDownLatch latch = new CountDownLatch(1);
         final AtomicReference<UpdateResponse> resp = new AtomicReference<>(null);
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java
index 230de44a8f6c5..f052ef79d82fb 100644
--- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java
@@ -371,7 +371,7 @@ public static ConnectorSearchResult getRandomConnectorSearchResult() {
             .build();
     }
 
-    private static ConnectorFeatures.FeatureEnabled randomConnectorFeatureEnabled() {
+    public static ConnectorFeatures.FeatureEnabled randomConnectorFeatureEnabled() {
         return new ConnectorFeatures.FeatureEnabled(randomBoolean());
     }
 
diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java
new file mode 100644
index 0000000000000..9a191dba2e525
--- /dev/null
+++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.application.connector.action;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.application.connector.ConnectorTestUtils;
+import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
+
+import java.io.IOException;
+
+public class UpdateConnectorFeaturesActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase<
+    UpdateConnectorFeaturesAction.Request> {
+
+    private String connectorId;
+
+    @Override
+    protected Writeable.Reader<UpdateConnectorFeaturesAction.Request> instanceReader() {
+        return UpdateConnectorFeaturesAction.Request::new;
+    }
+
+    @Override
+    protected UpdateConnectorFeaturesAction.Request createTestInstance() {
+        this.connectorId = randomUUID();
+        return new UpdateConnectorFeaturesAction.Request(connectorId, ConnectorTestUtils.getRandomConnectorFeatures());
+    }
+
+    @Override
+    protected UpdateConnectorFeaturesAction.Request mutateInstance(UpdateConnectorFeaturesAction.Request instance) throws IOException {
+        return randomValueOtherThan(instance, this::createTestInstance);
+    }
+
+    @Override
+    protected UpdateConnectorFeaturesAction.Request doParseInstance(XContentParser parser) throws IOException {
+        return UpdateConnectorFeaturesAction.Request.fromXContent(parser, this.connectorId);
+    }
+
+    @Override
+    protected UpdateConnectorFeaturesAction.Request mutateInstanceForVersion(
+        UpdateConnectorFeaturesAction.Request instance,
+        TransportVersion version
+    ) {
+        return instance;
+    }
+}
diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
index ae5af54f078dd..5561e14da980a 100644
--- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
+++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
@@ -133,6 +133,7 @@ public class Constants {
         "indices:data/write/xpack/connector/update_api_key_id",
         "indices:data/write/xpack/connector/update_configuration",
         "indices:data/write/xpack/connector/update_error",
+        "indices:data/write/xpack/connector/update_features",
         "indices:data/write/xpack/connector/update_filtering",
         "indices:data/write/xpack/connector/update_filtering/activate",
         "indices:data/write/xpack/connector/update_filtering/draft_validation",

From 8ac3e3dd909f3c5c04538e2ee08dfc7d5bc109c9 Mon Sep 17 00:00:00 2001
From: Rene Groeschke <rene@elastic.co>
Date: Tue, 4 Jun 2024 12:43:02 +0200
Subject: [PATCH 128/208] Update Gradle wrapper to 8.8 (#108021)

Fix incompatibility with 8.8 and our internal api usages

- Update ospackage to a version that contains a fix we provided
- Tweak build logic to avoid deprecation warnings
- Use newer permission api
- Use custom shadowplugin
- Rework ElasticsearchDistribution dependencies resolution
- Update Gradle wrapper to 8.8
---
 build-conventions/build.gradle                |  14 ++
 build-tools-internal/build.gradle             |   3 +
 .../gradle/wrapper/gradle-wrapper.properties  |   4 +-
 build-tools-internal/settings.gradle          |  12 +-
 ...nternalDistributionArchiveSetupPlugin.java |   4 +-
 .../internal/SymbolicLinkPreservingTar.java   |   6 +-
 .../internal/info/GlobalBuildInfoPlugin.java  |   2 +-
 .../DependencyLicensesPrecommitPlugin.java    |  25 ++--
 .../precommit/DependencyLicensesTask.java     |  24 +++-
 .../ThirdPartyAuditPrecommitPlugin.java       |  17 ++-
 .../internal/util/DependenciesUtils.java      |  49 +++++++
 .../gradle/internal/util/HdfsUtils.java       |  55 --------
 .../src/main/resources/minimumGradleVersion   |   2 +-
 build-tools/build.gradle                      |  12 ++
 build-tools/settings.gradle                   |   2 +-
 .../gradle/DistributionDownloadPlugin.java    | 132 ++++++++++--------
 .../gradle/ElasticsearchDistribution.java     |   7 +-
 .../testclusters/ElasticsearchNode.java       |  14 +-
 build.gradle                                  |  14 +-
 distribution/archives/build.gradle            |  26 +++-
 distribution/build.gradle                     |  24 +++-
 distribution/packages/build.gradle            |  74 ++++++----
 gradle/build.versions.toml                    |   2 +-
 gradle/verification-metadata.xml              |  32 +++--
 gradle/wrapper/gradle-wrapper.properties      |   4 +-
 gradlew                                       |   2 +-
 .../gradle/wrapper/gradle-wrapper.properties  |   4 +-
 server/build.gradle                           |  12 +-
 settings.gradle                               |   3 +
 29 files changed, 351 insertions(+), 230 deletions(-)
 create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java
 delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java

diff --git a/build-conventions/build.gradle b/build-conventions/build.gradle
index cd9a548a9901f..94b0312d0d5d3 100644
--- a/build-conventions/build.gradle
+++ b/build-conventions/build.gradle
@@ -8,6 +8,16 @@
 
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
+
+buildscript {
+  repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+    mavenCentral()
+  }
+}
+
 plugins {
     id 'java-gradle-plugin'
     id 'java-test-fixtures'
@@ -59,6 +69,10 @@ gradlePlugin {
 }
 
 repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+
     mavenCentral()
     gradlePluginPortal()
 }
diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle
index 52e72d973f2ed..84e56bbaf03ad 100644
--- a/build-tools-internal/build.gradle
+++ b/build-tools-internal/build.gradle
@@ -257,6 +257,9 @@ tasks.named('licenseHeaders').configure {
  *****************************************************************************/
 
 repositories {
+  maven {
+    url 'https://jitpack.io'
+  }
   mavenCentral()
   gradlePluginPortal()
 }
diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
index fcbbad6dd644c..515ab9d5f1822 100644
--- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
+++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
+distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/build-tools-internal/settings.gradle b/build-tools-internal/settings.gradle
index 6423750872ca2..1b4fb1215a59d 100644
--- a/build-tools-internal/settings.gradle
+++ b/build-tools-internal/settings.gradle
@@ -1,5 +1,13 @@
 pluginManagement {
-    includeBuild "../build-conventions"
+  repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+    mavenCentral()
+    gradlePluginPortal()
+  }
+
+  includeBuild "../build-conventions"
     includeBuild "../build-tools"
 }
 
@@ -9,4 +17,4 @@ dependencyResolutionManagement {
             from(files("../gradle/build.versions.toml"))
         }
     }
-}
\ No newline at end of file
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java
index bfc38e13043b9..d10cecf7fa50e 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java
@@ -99,8 +99,8 @@ private void configureGeneralTaskDefaults(Project project) {
         project.getTasks().withType(AbstractCopyTask.class).configureEach(t -> {
             t.dependsOn(project.getTasks().withType(EmptyDirTask.class));
             t.setIncludeEmptyDirs(true);
-            t.setDirMode(0755);
-            t.setFileMode(0644);
+            t.dirPermissions(permissions -> permissions.unix(0755));
+            t.filePermissions(permissions -> permissions.unix(0644));
         });
 
         // common config across all archives
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java
index 29c7dfd422547..52000e8c8fd71 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java
@@ -145,7 +145,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) {
                 visitedSymbolicLinks.add(details.getFile());
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString(), TarConstants.LF_SYMLINK);
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.LINK_FLAG | details.getMode());
+                entry.setMode(UnixStat.LINK_FLAG | details.getPermissions().toUnixNumeric());
                 try {
                     entry.setLinkName(Files.readSymbolicLink(details.getFile().toPath()).toString());
                     tar.putArchiveEntry(entry);
@@ -158,7 +158,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) {
             private void visitDirectory(final FileCopyDetailsInternal details) {
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString() + "/");
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.DIR_FLAG | details.getMode());
+                entry.setMode(UnixStat.DIR_FLAG | details.getPermissions().toUnixNumeric());
                 try {
                     tar.putArchiveEntry(entry);
                     tar.closeArchiveEntry();
@@ -170,7 +170,7 @@ private void visitDirectory(final FileCopyDetailsInternal details) {
             private void visitFile(final FileCopyDetailsInternal details) {
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString());
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.FILE_FLAG | details.getMode());
+                entry.setMode(UnixStat.FILE_FLAG | details.getPermissions().toUnixNumeric());
                 entry.setSize(details.getSize());
                 try {
                     tar.putArchiveEntry(entry);
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
index 5e62790a9d78a..42834928bafed 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
@@ -260,7 +260,7 @@ private List<JavaHome> getAvailableJavaVersions() {
     private Stream<InstallationLocation> getAvailableJavaInstallationLocationSteam() {
         return Stream.concat(
             javaInstallationRegistry.toolchains().stream().map(metadata -> metadata.location),
-            Stream.of(new InstallationLocation(Jvm.current().getJavaHome(), "Current JVM"))
+            Stream.of(InstallationLocation.userDefined(Jvm.current().getJavaHome(), "Current JVM"))
         );
     }
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java
index 72c08712a1fd9..b1d9cbd1f01d1 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java
@@ -12,30 +12,23 @@
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
-import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.artifacts.component.ComponentIdentifier;
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
 import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.TaskProvider;
 
 public class DependencyLicensesPrecommitPlugin extends PrecommitPlugin {
+    private static Spec<ComponentIdentifier> COMPONENT_FILTER = identifier -> (identifier instanceof ModuleComponentIdentifier)
+        && ((ModuleComponentIdentifier) identifier).getGroup().startsWith("org.elasticsearch") == false;
 
     @Override
     public TaskProvider<? extends Task> createTask(Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
-        TaskProvider<DependencyLicensesTask> dependencyLicenses = project.getTasks()
-            .register("dependencyLicenses", DependencyLicensesTask.class);
-
-        // only require dependency licenses for non-elasticsearch deps
-        dependencyLicenses.configure(t -> {
-            Configuration runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
-            Configuration compileOnly = project.getConfigurations()
-                .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
-            t.setDependencies(
-                runtimeClasspath.fileCollection(
-                    dependency -> dependency instanceof ProjectDependency == false
-                        && dependency.getGroup().startsWith("org.elasticsearch") == false
-                ).minus(compileOnly)
-            );
+        var dependencyLicenses = project.getTasks().register("dependencyLicenses", DependencyLicensesTask.class, t -> {
+            var runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
+            var compileOnly = project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
+            t.configureDependencies(runtimeClasspath, compileOnly, COMPONENT_FILTER);
         });
         return dependencyLicenses;
     }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java
index f71973c2fb15c..0099a4616f829 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java
@@ -11,6 +11,8 @@
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.InvalidUserDataException;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.component.ComponentIdentifier;
 import org.gradle.api.file.Directory;
 import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.FileCollection;
@@ -18,7 +20,9 @@
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
+import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
 import org.gradle.api.tasks.InputFiles;
@@ -41,6 +45,8 @@
 
 import javax.inject.Inject;
 
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView;
+
 /**
  * A task to check licenses for dependencies.
  * <p>
@@ -83,7 +89,7 @@
  * for the dependency. This artifact will be redistributed by us with the release to
  * comply with the license terms.
  */
-public class DependencyLicensesTask extends DefaultTask {
+public abstract class DependencyLicensesTask extends DefaultTask {
 
     private final Pattern regex = Pattern.compile("-v?\\d+.*");
 
@@ -181,6 +187,10 @@ public void ignoreFile(String file) {
         ignoreFiles.add(file);
     }
 
+    @Input
+    @Optional
+    public abstract Property<Spec<ComponentIdentifier>> getComponentFilter();
+
     @TaskAction
     public void checkDependencies() {
         if (dependencies == null) {
@@ -295,7 +305,6 @@ private String getFileName(String name, Map<String, ?> counters, String type) {
             // try the other suffix...TODO: get rid of this, just support ending in .txt
             return fileName + ".txt";
         }
-
         return fileName;
     }
 
@@ -310,4 +319,15 @@ public LinkedHashMap<String, String> getMappings() {
         return new LinkedHashMap<>(mappings);
     }
 
+    /**
+     * Convencience method for configuring dependencies to be checked and ignoring transitive dependencies for now.
+     * */
+    public void configureDependencies(
+        Configuration plusConfiguration,
+        Configuration minusConfiguration,
+        Spec<ComponentIdentifier> componentFilter
+    ) {
+        setDependencies(createFileCollectionFromNonTransitiveArtifactsView(plusConfiguration, componentFilter).minus(minusConfiguration));
+    }
+
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
index f6d3787a4f686..1fc030be42480 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
@@ -15,11 +15,14 @@
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
 import org.gradle.api.tasks.TaskProvider;
 
 import java.io.File;
 import java.nio.file.Path;
 
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView;
+
 public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
 
     public static final String JDK_JAR_HELL_CONFIG_NAME = "jdkJarHell";
@@ -54,12 +57,14 @@ public TaskProvider<? extends Task> createTask(Project project) {
             Configuration compileOnly = project.getConfigurations()
                 .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
             t.setClasspath(runtimeConfiguration.plus(compileOnly));
-            t.getJarsToScan().from(runtimeConfiguration.fileCollection(dep -> {
-                // These are SelfResolvingDependency, and some of them backed by file collections, like the Gradle API files,
-                // or dependencies added as `files(...)`, we can't be sure if those are third party or not.
-                // err on the side of scanning these to make sure we don't miss anything
-                return dep.getGroup() != null && dep.getGroup().startsWith("org.elasticsearch") == false;
-            }));
+            t.getJarsToScan()
+                .from(
+                    createFileCollectionFromNonTransitiveArtifactsView(
+                        runtimeConfiguration,
+                        identifier -> identifier instanceof ModuleComponentIdentifier
+                            && ((ModuleComponentIdentifier) identifier).getGroup().startsWith("org.elasticsearch") == false
+                    )
+                );
             t.dependsOn(resourcesTask);
             if (BuildParams.getIsRuntimeJavaHomeSet()) {
                 t.getJavaHome().set(project.provider(BuildParams::getRuntimeJavaHome).map(File::getPath));
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java
new file mode 100644
index 0000000000000..081c28c14fd91
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.util;
+
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.ResolvableDependencies;
+import org.gradle.api.artifacts.component.ComponentIdentifier;
+import org.gradle.api.artifacts.result.ResolvedComponentResult;
+import org.gradle.api.artifacts.result.ResolvedDependencyResult;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.specs.AndSpec;
+import org.gradle.api.specs.Spec;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class DependenciesUtils {
+
+    public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
+        Configuration configuration,
+        Spec<ComponentIdentifier> componentFilter
+    ) {
+        ResolvableDependencies incoming = configuration.getIncoming();
+        return incoming.artifactView(viewConfiguration -> {
+            Set<ComponentIdentifier> firstLevelDependencyComponents = incoming.getResolutionResult()
+                .getRootComponent()
+                .map(
+                    rootComponent -> rootComponent.getDependencies()
+                        .stream()
+                        .filter(dependency -> dependency instanceof ResolvedDependencyResult)
+                        .map(dependency -> (ResolvedDependencyResult) dependency)
+                        .filter(dependency -> dependency.getSelected() instanceof ResolvedComponentResult)
+                        .map(dependency -> dependency.getSelected().getId())
+                        .collect(Collectors.toSet())
+                )
+                .get();
+            viewConfiguration.componentFilter(
+                new AndSpec<>(identifier -> firstLevelDependencyComponents.contains(identifier), componentFilter)
+            );
+        }).getFiles();
+    }
+
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java
deleted file mode 100644
index 8b9570d62389e..0000000000000
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.internal.util;
-
-import org.elasticsearch.gradle.OS;
-import org.gradle.api.Project;
-import org.gradle.api.logging.Logging;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-public class HdfsUtils {
-
-    public static boolean isHdfsFixtureSupported(Project project) {
-        String projectPath = project.getProjectDir().getPath();
-        if (isLegalHdfsPath(projectPath) == false) {
-            Logging.getLogger(HdfsUtils.class).warn("hdfs Fixture unsupported since there are spaces in the path: '" + projectPath + "'");
-            return false;
-        }
-        return (OS.current() != OS.WINDOWS) ? true : isHadoopWindowsInstallationAvailable();
-    }
-
-    private static boolean isHadoopWindowsInstallationAvailable() {
-        // hdfs fixture will not start without hadoop native libraries on windows
-        String nativePath = System.getenv("HADOOP_HOME");
-        if (nativePath != null) {
-            Path path = Paths.get(nativePath);
-            if (Files.isDirectory(path)
-                && Files.exists(path.resolve("bin").resolve("winutils.exe"))
-                && Files.exists(path.resolve("bin").resolve("hadoop.dll"))
-                && Files.exists(path.resolve("bin").resolve("hdfs.dll"))) {
-                return true;
-            } else {
-                throw new IllegalStateException(
-                    "HADOOP_HOME: " + path + " is invalid, does not contain hadoop native libraries in \\$HADOOP_HOME\\bin"
-                );
-            }
-        }
-        Logging.getLogger(HdfsUtils.class).warn("hdfs Fixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH");
-
-        return false;
-    }
-
-    public static boolean isLegalHdfsPath(String path) {
-        return path.contains(" ") == false;
-
-    }
-}
diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion
index 631c6d36a93a4..83ea3179ddacc 100644
--- a/build-tools-internal/src/main/resources/minimumGradleVersion
+++ b/build-tools-internal/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-8.7
\ No newline at end of file
+8.8
\ No newline at end of file
diff --git a/build-tools/build.gradle b/build-tools/build.gradle
index eb5573ac03e0e..7ba5e9f6faa62 100644
--- a/build-tools/build.gradle
+++ b/build-tools/build.gradle
@@ -6,6 +6,15 @@
  * Side Public License, v 1.
  */
 
+buildscript {
+  repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+    mavenCentral()
+  }
+}
+
 plugins {
     id 'java-gradle-plugin'
     id 'groovy'
@@ -107,6 +116,9 @@ configurations {
 }
 
 repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
     mavenCentral()
     gradlePluginPortal()
 }
diff --git a/build-tools/settings.gradle b/build-tools/settings.gradle
index 63d80efcd505e..7590b8b6b054e 100644
--- a/build-tools/settings.gradle
+++ b/build-tools/settings.gradle
@@ -17,4 +17,4 @@ dependencyResolutionManagement {
             from(files("../gradle/build.versions.toml"))
         }
     }
-}
\ No newline at end of file
+}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
index fb8416b24d052..2bc4aa1a1be36 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
@@ -11,11 +11,9 @@
 import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes;
 import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform;
 import org.elasticsearch.gradle.transform.UnzipTransform;
-import org.gradle.api.Action;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
-import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.dsl.DependencyHandler;
 import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
 import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
@@ -24,6 +22,7 @@
 import org.gradle.api.provider.Provider;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import javax.inject.Inject;
@@ -46,6 +45,7 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
     public static final String DISTRO_EXTRACTED_CONFIG_PREFIX = "es_distro_extracted_";
     public static final String DISTRO_CONFIG_PREFIX = "es_distro_file_";
 
+    private final ObjectFactory objectFactory;
     private NamedDomainObjectContainer<ElasticsearchDistribution> distributionsContainer;
     private List<DistributionResolution> distributionsResolutionStrategies;
 
@@ -53,6 +53,7 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
 
     @Inject
     public DistributionDownloadPlugin(ObjectFactory objectFactory) {
+        this.objectFactory = objectFactory;
         this.dockerAvailability = objectFactory.property(Boolean.class).value(false);
     }
 
@@ -67,36 +68,92 @@ public void apply(Project project) {
             transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
         });
 
-        ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz");
+        var tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz");
         project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> {
             transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, tarArtifactTypeDefinition.getName());
             transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
         });
 
         setupResolutionsContainer(project);
-        setupDistributionContainer(project, dockerAvailability);
+        setupDistributionContainer(project);
         setupDownloadServiceRepo(project);
     }
 
-    private void setupDistributionContainer(Project project, Property<Boolean> dockerAvailable) {
-
+    private void setupDistributionContainer(Project project) {
         distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
-            Configuration fileConfiguration = project.getConfigurations().create(DISTRO_CONFIG_PREFIX + name);
-            Configuration extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name);
+            var fileConfiguration = project.getConfigurations().create(DISTRO_CONFIG_PREFIX + name);
+            var extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name);
             extractedConfiguration.getAttributes()
                 .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
-            return new ElasticsearchDistribution(
+
+            var distribution = new ElasticsearchDistribution(
                 name,
-                project.getObjects(),
+                objectFactory,
                 dockerAvailability,
-                project.getObjects().fileCollection().from(fileConfiguration),
-                project.getObjects().fileCollection().from(extractedConfiguration),
-                new FinalizeDistributionAction(distributionsResolutionStrategies, project)
+                objectFactory.fileCollection().from(fileConfiguration),
+                objectFactory.fileCollection().from(extractedConfiguration)
             );
+
+            registerDistributionDependencies(project, distribution);
+            return distribution;
         });
         project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
     }
 
+    private void registerDistributionDependencies(Project project, ElasticsearchDistribution distribution) {
+        project.getConfigurations()
+            .getByName(DISTRO_CONFIG_PREFIX + distribution.getName())
+            .getDependencies()
+            .addLater(
+                project.provider(() -> distribution.maybeFreeze())
+                    .map(
+                        frozenDistro -> project.getDependencies()
+                            .create(resolveDependencyNotation(project, frozenDistro).getDefaultNotation())
+                    )
+            );
+
+        project.getConfigurations()
+            .getByName(DISTRO_EXTRACTED_CONFIG_PREFIX + distribution.getName())
+            .getDependencies()
+            .addAllLater(
+                project.provider(() -> distribution.maybeFreeze())
+                    .map(
+                        frozenDistro -> distribution.getType().shouldExtract()
+                            ? List.of(
+                                project.getDependencies().create(resolveDependencyNotation(project, frozenDistro).getExtractedNotation())
+                            )
+                            : Collections.emptyList()
+                    )
+            );
+    }
+
+    private DistributionDependency resolveDependencyNotation(Project project, ElasticsearchDistribution distro) {
+        return distributionsResolutionStrategies.stream()
+            .map(r -> r.getResolver().resolve(project, distro))
+            .filter(d -> d != null)
+            .findFirst()
+            .orElseGet(() -> DistributionDependency.of(dependencyNotation(distro)));
+    }
+
+    /**
+     * Returns a dependency object representing the given distribution.
+     * <p>
+     * The returned object is suitable to be passed to {@link DependencyHandler}.
+     * The concrete type of the object will be a set of maven coordinates as a {@link String}.
+     * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial
+     * coordinates that resolve to the Elastic download service through an ivy repository.
+     */
+    private static String dependencyNotation(ElasticsearchDistribution distribution) {
+        if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) {
+            return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
+        }
+        var distroVersion = Version.fromString(distribution.getVersion());
+        var extension = distribution.getType().getExtension(distribution.getPlatform());
+        var classifier = distribution.getType().getClassifier(distribution.getPlatform(), distroVersion);
+        var group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
+        return group + ":elasticsearch" + ":" + distribution.getVersion() + classifier + "@" + extension;
+    }
+
     private void setupResolutionsContainer(Project project) {
         distributionsResolutionStrategies = new ArrayList<>();
         project.getExtensions().add(RESOLUTION_CONTAINER_NAME, distributionsResolutionStrategies);
@@ -133,53 +190,4 @@ private static void setupDownloadServiceRepo(Project project) {
         addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP);
     }
 
-    private record FinalizeDistributionAction(List<DistributionResolution> resolutionList, Project project)
-        implements
-            Action<ElasticsearchDistribution> {
-        @Override
-
-        public void execute(ElasticsearchDistribution distro) {
-            finalizeDistributionDependencies(project, distro);
-        }
-
-        private void finalizeDistributionDependencies(Project project, ElasticsearchDistribution distribution) {
-            // for the distribution as a file, just depend on the artifact directly
-            DistributionDependency distributionDependency = resolveDependencyNotation(project, distribution);
-            project.getDependencies().add(DISTRO_CONFIG_PREFIX + distribution.getName(), distributionDependency.getDefaultNotation());
-            // no extraction needed for rpm, deb or docker
-            if (distribution.getType().shouldExtract()) {
-                // The extracted configuration depends on the artifact directly but has
-                // an artifact transform registered to resolve it as an unpacked folder.
-                project.getDependencies()
-                    .add(DISTRO_EXTRACTED_CONFIG_PREFIX + distribution.getName(), distributionDependency.getExtractedNotation());
-            }
-        }
-
-        private DistributionDependency resolveDependencyNotation(Project project, ElasticsearchDistribution distro) {
-            return resolutionList.stream()
-                .map(r -> r.getResolver().resolve(project, distro))
-                .filter(d -> d != null)
-                .findFirst()
-                .orElseGet(() -> DistributionDependency.of(dependencyNotation(distro)));
-        }
-
-        /**
-         * Returns a dependency object representing the given distribution.
-         * <p>
-         * The returned object is suitable to be passed to {@link DependencyHandler}.
-         * The concrete type of the object will be a set of maven coordinates as a {@link String}.
-         * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial
-         * coordinates that resolve to the Elastic download service through an ivy repository.
-         */
-        private String dependencyNotation(ElasticsearchDistribution distribution) {
-            if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) {
-                return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
-            }
-            Version distroVersion = Version.fromString(distribution.getVersion());
-            String extension = distribution.getType().getExtension(distribution.getPlatform());
-            String classifier = distribution.getType().getClassifier(distribution.getPlatform(), distroVersion);
-            String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
-            return group + ":elasticsearch" + ":" + distribution.getVersion() + classifier + "@" + extension;
-        }
-    }
 }
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
index fab6926008d6c..afb90ba1ca62e 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.gradle;
 
 import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes;
-import org.gradle.api.Action;
 import org.gradle.api.Buildable;
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileCollection;
@@ -55,7 +54,6 @@ public String toString() {
     private final Property<Boolean> failIfUnavailable;
     private final Property<Boolean> preferArchive;
     private final ConfigurableFileCollection extracted;
-    private transient Action<ElasticsearchDistribution> distributionFinalizer;
     private boolean frozen = false;
 
     ElasticsearchDistribution(
@@ -63,8 +61,7 @@ public String toString() {
         ObjectFactory objectFactory,
         Property<Boolean> dockerAvailability,
         ConfigurableFileCollection fileConfiguration,
-        ConfigurableFileCollection extractedConfiguration,
-        Action<ElasticsearchDistribution> distributionFinalizer
+        ConfigurableFileCollection extractedConfiguration
     ) {
         this.name = name;
         this.dockerAvailability = dockerAvailability;
@@ -78,7 +75,6 @@ public String toString() {
         this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true);
         this.preferArchive = objectFactory.property(Boolean.class).convention(false);
         this.extracted = extractedConfiguration;
-        this.distributionFinalizer = distributionFinalizer;
     }
 
     public String getName() {
@@ -172,7 +168,6 @@ public String toString() {
     public ElasticsearchDistribution maybeFreeze() {
         if (frozen == false) {
             finalizeValues();
-            distributionFinalizer.execute(this);
             frozen = true;
         }
         return this;
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
index 999f27a646b1f..d25798ad071bd 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@@ -246,14 +246,12 @@ public void setVersions(List<String> versions) {
     private void doSetVersion(String version) {
         String distroName = "testclusters" + path.replace(":", "-") + "-" + this.name + "-" + version;
         NamedDomainObjectContainer<ElasticsearchDistribution> container = DistributionDownloadPlugin.getContainer(project);
-        if (container.findByName(distroName) == null) {
-            container.create(distroName);
-        }
-        ElasticsearchDistribution distro = container.getByName(distroName);
-        distro.setVersion(version);
-        distro.setArchitecture(Architecture.current());
-        setDistributionType(distro, testDistribution);
-        distributions.add(distro);
+        // TODO Refactor test using register<> for reducing overhead
+        ElasticsearchDistribution distribution = container.maybeCreate(distroName);
+        distribution.setVersion(version);
+        distribution.setArchitecture(Architecture.current());
+        setDistributionType(distribution, testDistribution);
+        distributions.add(distribution);
     }
 
     @Internal
diff --git a/build.gradle b/build.gradle
index 1d9757f32543d..3869d21b49bfe 100644
--- a/build.gradle
+++ b/build.gradle
@@ -25,6 +25,16 @@ import java.nio.file.Files
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
 
+buildscript {
+  repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+
+    mavenCentral()
+  }
+}
+
 plugins {
   id 'lifecycle-base'
   id 'elasticsearch.docker-support'
@@ -325,7 +335,7 @@ allprojects {
       integTestTask.mustRunAfter tasks.matching { it.name.equals("test") }
     }
 
-    configurations.matching { it.canBeResolved }.all { Configuration configuration ->
+/*    configurations.matching { it.canBeResolved }.all { Configuration configuration ->
       dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
         Project upstreamProject = dep.dependencyProject
         if (project.path != upstreamProject?.path) {
@@ -336,7 +346,7 @@ allprojects {
           }
         }
       }
-    }
+    }*/
   }
 
   apply plugin: 'elasticsearch.formatting'
diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
index 4d7850477dbf5..815ac5d4c2dd8 100644
--- a/distribution/archives/build.gradle
+++ b/distribution/archives/build.gradle
@@ -18,11 +18,17 @@ CopySpec archiveFiles(String distributionType, String os, String architecture, b
         with libFiles(os, architecture)
       }
       into('config') {
-        dirMode 0750
-        fileMode 0660
+        dirPermissions {
+          unix 0750
+        }
+        filePermissions {
+          unix 0660
+        }
         with configFiles(distributionType, isTestDistro)
         from {
-          dirMode 0750
+          dirPermissions {
+            unix 0750
+          }
           jvmOptionsDir.getParent()
         }
       }
@@ -36,21 +42,31 @@ CopySpec archiveFiles(String distributionType, String os, String architecture, b
       }
       into('') {
         from {
-          dirMode 0755
+          dirPermissions {
+            unix 0755
+          }
           logsDir.getParent()
         }
       }
       into('') {
         from {
-          dirMode 0755
+          dirPermissions {
+            unix 0755
+          }
           pluginsDir.getParent()
         }
       }
       from(rootProject.projectDir) {
+        filePermissions {
+          unix(0644)
+        }
         include 'README.asciidoc'
       }
       from(rootProject.file('licenses')) {
         include isTestDistro ? 'SSPL-1.0+ELASTIC-LICENSE-2.0.txt' : 'ELASTIC-LICENSE-2.0.txt'
+        filePermissions {
+          unix(0644)
+        }
         rename { 'LICENSE.txt' }
       }
 
diff --git a/distribution/build.gradle b/distribution/build.gradle
index c3f9192ecee05..77f1a2d032c73 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -346,9 +346,9 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           if (it.relativePath.segments[-2] == 'bin' || (os == 'darwin' && it.relativePath.segments[-2] == 'MacOS')) {
             // bin files, wherever they are within modules (eg platform specific) should be executable
             // and MacOS is an alternative to bin on macOS
-            it.mode = 0755
+            it.permissions.unix(0755)
           } else {
-            it.mode = 0644
+            it.permissions.unix(0644)
           }
         }
         List excludePlatforms = ['linux-x86_64', 'linux-aarch64', 'windows-x86_64', 'darwin-x86_64', 'darwin-aarch64']
@@ -404,7 +404,11 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           from '../src/bin'
           exclude '*.exe'
           exclude '*.bat'
-          eachFile { it.setMode(0755) }
+          eachFile {
+            it.permissions{
+              unix(0755)
+            }
+          }
           filter("tokens" : expansionsForDistribution(distributionType, testDistro), ReplaceTokens.class)
         }
         // windows files, only for zip
@@ -422,7 +426,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         }
         // module provided bin files
         with copySpec {
-          eachFile { it.setMode(0755) }
+          eachFile { it.permissions.unix(0755) }
           from(testDistro ? integTestBinFiles : defaultBinFiles)
           if (distributionType != 'zip') {
             exclude '*.bat'
@@ -437,7 +441,9 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           from buildServerNoticeTaskProvider
         } else {
           from (buildDefaultNoticeTaskProvider) {
-            fileMode = 0644
+            filePermissions {
+              unix(0644)
+            }
           }
         }
       }
@@ -456,7 +462,13 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         }
         eachFile { FileCopyDetails details ->
           if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
-            details.mode = 0755
+            details.permissions {
+              unix(0755)
+            }
+          } else {
+            details.permissions {
+              unix(0644)
+            }
           }
           if (details.name == 'src.zip') {
             details.exclude()
diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle
index 6b57f32310c93..6c31bc44017c3 100644
--- a/distribution/packages/build.gradle
+++ b/distribution/packages/build.gradle
@@ -42,20 +42,10 @@ import java.util.regex.Pattern
  *    dpkg -c path/to/elasticsearch.deb
  */
 
-buildscript {
-  repositories {
-    maven {
-      url 'https://jitpack.io'
-    }
-    mavenCentral()
-  }
-  dependencies {
-    classpath "com.github.breskeby:gradle-ospackage-plugin:2da19425133"
-  }
+plugins {
+  id "com.netflix.nebula.ospackage-base" version "11.9.1"
 }
 
-apply plugin: "com.netflix.nebula.ospackage-base"
-
 ['deb', 'rpm'].each { type ->
   String packagingFiles = "build/packaging/${type}"
 
@@ -138,7 +128,9 @@ def commonPackageConfig(String type, String architecture) {
       }
       from(rootProject.projectDir) {
         include 'README.asciidoc'
-        fileMode 0644
+        filePermissions {
+          unix 0644
+        }
       }
       into('lib') {
         with libFiles('linux', architecture)
@@ -159,9 +151,13 @@ def commonPackageConfig(String type, String architecture) {
             directory('/' + segments[0..i].join('/'), 0755)
           }
           if (segments[-2] == 'bin' || segments[-1] == 'jspawnhelper') {
-            fcp.mode = 0755
+            fcp.permissions {
+              unix(0755)
+            }
           } else {
-            fcp.mode = 0644
+            fcp.permissions {
+              unix(0644)
+            }
           }
         }
       }
@@ -171,7 +167,9 @@ def commonPackageConfig(String type, String architecture) {
     if (type == 'deb') {
       into("/usr/share/doc/${packageName}") {
         from "${packagingFiles}/copyright"
-        fileMode 0644
+        filePermissions {
+          unix(0644)
+        }
       }
     } else {
       assert type == 'rpm'
@@ -180,7 +178,9 @@ def commonPackageConfig(String type, String architecture) {
           include 'ELASTIC-LICENSE-2.0.txt'
           rename { 'LICENSE.txt' }
         }
-        fileMode 0644
+        filePermissions {
+          unix(0644)
+        }
       }
     }
 
@@ -194,7 +194,9 @@ def commonPackageConfig(String type, String architecture) {
     configurationFile '/etc/elasticsearch/users'
     configurationFile '/etc/elasticsearch/users_roles'
     from("${packagingFiles}") {
-      dirMode 02750
+      dirPermissions {
+        unix(02750)
+      }
       into('/etc')
       permissionGroup 'elasticsearch'
       setgid true
@@ -205,9 +207,13 @@ def commonPackageConfig(String type, String architecture) {
     }
     from("${packagingFiles}/etc/elasticsearch") {
       into('/etc/elasticsearch')
-      dirMode 02750
+      dirPermissions {
+        unix(02750)
+      }
       setgid = true
-      fileMode 0660
+      filePermissions {
+        unix(0660)
+      }
       permissionGroup 'elasticsearch'
       includeEmptyDirs true
       createDirectoryEntry true
@@ -218,28 +224,38 @@ def commonPackageConfig(String type, String architecture) {
     into(new File(envFile).getParent()) {
       fileType CONFIG | NOREPLACE
       permissionGroup 'elasticsearch'
-      fileMode 0660
+      filePermissions {
+        unix(0660)
+      }
       from "${packagingFiles}/env/elasticsearch"
     }
 
     // ========= systemd =========
     into('/usr/lib/tmpfiles.d') {
       from "${packagingFiles}/systemd/elasticsearch.conf"
-      fileMode 0644
+      filePermissions {
+        unix(0644)
+      }
     }
     into('/usr/lib/systemd/system') {
       fileType CONFIG | NOREPLACE
       from "${packagingFiles}/systemd/elasticsearch.service"
-      fileMode 0644
+      filePermissions {
+        unix(0644)
+      }
     }
     into('/usr/lib/sysctl.d') {
       fileType CONFIG | NOREPLACE
       from "${packagingFiles}/systemd/sysctl/elasticsearch.conf"
-      fileMode 0644
+      filePermissions {
+        unix(0644)
+      }
     }
     into('/usr/share/elasticsearch/bin') {
       from "${packagingFiles}/systemd/systemd-entrypoint"
-      fileMode 0755
+      filePermissions {
+        unix(0755)
+      }
     }
 
     // ========= empty dirs =========
@@ -253,7 +269,9 @@ def commonPackageConfig(String type, String architecture) {
         createDirectoryEntry true
         user u
         permissionGroup g
-        dirMode = mode
+        dirPermissions {
+          unix(mode)
+        }
         setgid (mode == 02750)
       }
     }
@@ -322,7 +340,9 @@ Closure commonDebConfig(String architecture) {
 
     into('/usr/share/lintian/overrides') {
       from('src/deb/lintian/elasticsearch')
-      fileMode 0644
+      filePermissions {
+        unix(0644)
+      }
     }
   }
 }
diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml
index 6b5a541d15661..ba81673120569 100644
--- a/gradle/build.versions.toml
+++ b/gradle/build.versions.toml
@@ -38,7 +38,7 @@ maven-model = "org.apache.maven:maven-model:3.6.2"
 mockito-core = "org.mockito:mockito-core:1.9.5"
 nebula-info = "com.netflix.nebula:gradle-info-plugin:11.3.3"
 reflections = "org.reflections:reflections:0.9.12"
-shadow-plugin = "com.github.johnrengelman:shadow:8.1.1"
+shadow-plugin = "com.github.breskeby:shadow:3b035f2"
 spock-core = { group = "org.spockframework", name="spock-core", version.ref="spock" }
 spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" }
 spock-platform = { group = "org.spockframework", name="spock-bom", version.ref="spock" }
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index 41c3bafde5e33..532112d0138d3 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -376,9 +376,14 @@
             <sha256 value="d6eb9f2f40049a7a808baf11ffba0737648e62ff52fde9271d808e5d57a27279" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.github.breskeby" name="gradle-ospackage-plugin" version="2da19425133">
-         <artifact name="gradle-ospackage-plugin-2da19425133.jar">
-            <sha256 value="452e01bc961259e2a0f5d6f193a03cc8d29f560a433f7c8066158c97d3327af9" origin="Generated by Gradle"/>
+      <component group="com.github.breskeby" name="shadow" version="3b035f2">
+         <artifact name="shadow-3b035f2.jar">
+            <sha256 value="fb53bbff0eeb6835ad0712d22d945ad49524922956dd1d89bd487344d228f339" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.github.breskeby.shadow" name="shadow" version="3b035f2">
+         <artifact name="shadow-3b035f2.jar">
+            <sha256 value="9c3ca9918ae9fc42bc7e4d5f5259ec902ca6f1be06aaa6e66e94c21dcef4a617" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.github.docker-java" name="docker-java-api" version="3.3.4">
@@ -851,14 +856,9 @@
             <sha256 value="d694edd7bae3bc1a8e0dae2f5a22c479ff04d6b9bfcb0ab751a42f02e02d2100" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.netflix.nebula" name="gradle-ospackage-plugin" version="11.4.0">
-         <artifact name="gradle-ospackage-plugin-11.4.0.jar">
-            <sha256 value="78ef9c5162bf857a24b774b46c09703f2d316575a7b0363a69e05eb237cdd376" origin="Manually"/>
-         </artifact>
-      </component>
-      <component group="com.netflix.nebula" name="gradle-ospackage-plugin" version="11.5.0">
-         <artifact name="gradle-ospackage-plugin-11.5.0.jar">
-            <sha256 value="47cac89b99bbc9a9d215483ff05418464cc7b2d3dcf3eddd39a90bb4e91c1c2f" origin="Generated by Gradle"/>
+      <component group="com.netflix.nebula" name="gradle-ospackage-plugin" version="11.9.1">
+         <artifact name="gradle-ospackage-plugin-11.9.1.jar">
+            <sha256 value="63b2c387cb83481bb8d7e33e0be02e9000c0ec30843ca7730641392f3f5f28ad" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.netflix.nebula" name="nebula-gradle-interop" version="2.0.0">
@@ -4077,6 +4077,11 @@
             <sha256 value="0c128a9ec3f33c98959272f6d16cf14247b508f58951574bcdbd2b56d6326364" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2.asm" name="asm-commons" version="9.7">
+         <artifact name="asm-commons-9.7.jar">
+            <sha256 value="389bc247958e049fc9a0408d398c92c6d370c18035120395d4cba1d9d9304b7a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.ow2.asm" name="asm-tree" version="7.2">
          <artifact name="asm-tree-7.2.jar">
             <sha256 value="c063f5a67fa03cdc9bd79fd1c2ea6816cc4a19473ecdfbd9e9153b408c6f2656" origin="Generated by Gradle"/>
@@ -4182,6 +4187,11 @@
             <sha256 value="d3ef575e3e4979678dc01bf1dcce51021493b4d11fb7f1be8ad982877c16a1c0" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.slf4j" name="slf4j-api" version="1.7.5">
+         <artifact name="slf4j-api-1.7.5.jar">
+            <sha256 value="fe30825245d2336c859dc38d60c0fc5f3668dbf29cd586828d2b5667ec355b91" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.slf4j" name="slf4j-api" version="2.0.10">
          <artifact name="slf4j-api-2.0.10.jar">
             <sha256 value="b7ddb31a515debbddec8e9145e2cf7b197926f40e454376647724f92e6382043" origin="Generated by Gradle"/>
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index fcbbad6dd644c..515ab9d5f1822 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
+distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/gradlew b/gradlew
index 1aa94a4269074..b740cf13397ab 100755
--- a/gradlew
+++ b/gradlew
@@ -55,7 +55,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties
index fcbbad6dd644c..515ab9d5f1822 100644
--- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties
+++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
+distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/server/build.gradle b/server/build.gradle
index 03713bc3d2837..5831930421c60 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -254,12 +254,12 @@ tasks.named("thirdPartyAudit").configure {
 tasks.named("dependencyLicenses").configure {
     mapping from: /lucene-.*/, to: 'lucene'
     mapping from: /log4j-.*/, to: 'log4j'
-    dependencies = project.configurations.runtimeClasspath.fileCollection {
-        it.group.startsWith('org.elasticsearch') == false ||
-                // keep the following org.elasticsearch jars in
-                (it.name == 'jna' ||
-                        it.name == 'securesm')
-    }
+
+    configureDependencies(
+      project.configurations.runtimeClasspath, project.configurations.resolveableCompileOnly, identifier -> {
+      return identifier instanceof ModuleComponentIdentifier
+        (identifier.moduleIdentifier.name == 'jna' || identifier.moduleIdentifier.name == 'securesm')
+    })
 }
 
 tasks.named("licenseHeaders").configure {
diff --git a/settings.gradle b/settings.gradle
index 48e3794c9005d..6ed340b27da65 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -4,6 +4,9 @@ import org.elasticsearch.gradle.internal.toolchain.AdoptiumJdkToolchainResolver
 
 pluginManagement {
   repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
     mavenCentral()
     gradlePluginPortal()
   }

From c2e3505dbb5424ae2d1c04997d4b13627ecd113e Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Tue, 4 Jun 2024 08:23:51 -0400
Subject: [PATCH 129/208] [ESQL] Dependency Injection for the Warnings
 Collector (#108927)

This lays the groundwork for fixing #108519, although I don't intend to fix it in this PR. In order to address that issue, we need to be able to control at call time how we deal with warnings, but, prior to this work, the ExpressionEvaluators all hard code their warnings implementation.

This PR will allow injecting in a different implementation when we create the evaluator. Unfortunately, this has fallen slightly short of that ambition. We need to be able to construct evaluators from within compute, but the Warnings class, and it's Source dependency, are in the esql package, which we don't want to import into compute. As result, I've pared this back to just passing in a toggle for warnings collection, via a no-op collector implementation. This is sufficient to solve the linked issue (still not done in this PR) and preserves our current encapsulation, while leaving a clear path to further refine how we control warnings in the future.

Note that this is changing the code generation, which means it will touch all of the generated files, which are still in source control. The actual change is much smaller than the diff would imply.

Replaces #108635
---
 .../compute/gen/EvaluatorImplementer.java     |  2 +-
 .../compute/gen/MvEvaluatorImplementer.java   |  4 +--
 .../compute/operator/DriverContext.java       | 32 +++++++++++++++++++
 .../operator/logical/NotEvaluator.java        |  2 +-
 .../conditional/GreatestBooleanEvaluator.java |  2 +-
 .../GreatestBytesRefEvaluator.java            |  2 +-
 .../conditional/GreatestDoubleEvaluator.java  |  2 +-
 .../conditional/GreatestIntEvaluator.java     |  2 +-
 .../conditional/GreatestLongEvaluator.java    |  2 +-
 .../conditional/LeastBooleanEvaluator.java    |  2 +-
 .../conditional/LeastBytesRefEvaluator.java   |  2 +-
 .../conditional/LeastDoubleEvaluator.java     |  2 +-
 .../scalar/conditional/LeastIntEvaluator.java |  2 +-
 .../conditional/LeastLongEvaluator.java       |  2 +-
 .../scalar/convert/FromBase64Evaluator.java   |  2 +-
 .../scalar/convert/ToBase64Evaluator.java     |  2 +-
 .../date/DateDiffConstantEvaluator.java       |  2 +-
 .../scalar/date/DateDiffEvaluator.java        |  2 +-
 .../date/DateExtractConstantEvaluator.java    |  2 +-
 .../scalar/date/DateExtractEvaluator.java     |  2 +-
 .../date/DateFormatConstantEvaluator.java     |  2 +-
 .../scalar/date/DateFormatEvaluator.java      |  2 +-
 .../date/DateParseConstantEvaluator.java      |  2 +-
 .../scalar/date/DateParseEvaluator.java       |  2 +-
 .../scalar/date/DateTruncEvaluator.java       |  2 +-
 .../function/scalar/date/NowEvaluator.java    |  2 +-
 .../scalar/ip/CIDRMatchEvaluator.java         |  2 +-
 .../scalar/math/AbsDoubleEvaluator.java       |  2 +-
 .../function/scalar/math/AbsIntEvaluator.java |  2 +-
 .../scalar/math/AbsLongEvaluator.java         |  2 +-
 .../function/scalar/math/AcosEvaluator.java   |  2 +-
 .../function/scalar/math/AsinEvaluator.java   |  2 +-
 .../function/scalar/math/Atan2Evaluator.java  |  2 +-
 .../function/scalar/math/AtanEvaluator.java   |  2 +-
 .../scalar/math/CastIntToDoubleEvaluator.java |  2 +-
 .../scalar/math/CastIntToLongEvaluator.java   |  2 +-
 .../math/CastIntToUnsignedLongEvaluator.java  |  2 +-
 .../math/CastLongToDoubleEvaluator.java       |  2 +-
 .../math/CastLongToUnsignedLongEvaluator.java |  2 +-
 .../CastUnsignedLongToDoubleEvaluator.java    |  2 +-
 .../scalar/math/CbrtDoubleEvaluator.java      |  2 +-
 .../scalar/math/CbrtIntEvaluator.java         |  2 +-
 .../scalar/math/CbrtLongEvaluator.java        |  2 +-
 .../math/CbrtUnsignedLongEvaluator.java       |  2 +-
 .../scalar/math/CeilDoubleEvaluator.java      |  2 +-
 .../function/scalar/math/CosEvaluator.java    |  2 +-
 .../function/scalar/math/CoshEvaluator.java   |  2 +-
 .../scalar/math/FloorDoubleEvaluator.java     |  2 +-
 .../scalar/math/Log10DoubleEvaluator.java     |  2 +-
 .../scalar/math/Log10IntEvaluator.java        |  2 +-
 .../scalar/math/Log10LongEvaluator.java       |  2 +-
 .../math/Log10UnsignedLongEvaluator.java      |  2 +-
 .../scalar/math/LogConstantEvaluator.java     |  2 +-
 .../function/scalar/math/LogEvaluator.java    |  2 +-
 .../function/scalar/math/PowEvaluator.java    |  2 +-
 .../scalar/math/RoundDoubleEvaluator.java     |  2 +-
 .../math/RoundDoubleNoDecimalsEvaluator.java  |  2 +-
 .../scalar/math/RoundIntEvaluator.java        |  2 +-
 .../scalar/math/RoundLongEvaluator.java       |  2 +-
 .../math/RoundUnsignedLongEvaluator.java      |  2 +-
 .../scalar/math/SignumDoubleEvaluator.java    |  2 +-
 .../scalar/math/SignumIntEvaluator.java       |  2 +-
 .../scalar/math/SignumLongEvaluator.java      |  2 +-
 .../math/SignumUnsignedLongEvaluator.java     |  2 +-
 .../function/scalar/math/SinEvaluator.java    |  2 +-
 .../function/scalar/math/SinhEvaluator.java   |  2 +-
 .../scalar/math/SqrtDoubleEvaluator.java      |  2 +-
 .../scalar/math/SqrtIntEvaluator.java         |  2 +-
 .../scalar/math/SqrtLongEvaluator.java        |  2 +-
 .../math/SqrtUnsignedLongEvaluator.java       |  2 +-
 .../function/scalar/math/TanEvaluator.java    |  2 +-
 .../function/scalar/math/TanhEvaluator.java   |  2 +-
 .../multivalue/MvSliceBooleanEvaluator.java   |  2 +-
 .../multivalue/MvSliceBytesRefEvaluator.java  |  2 +-
 .../multivalue/MvSliceDoubleEvaluator.java    |  2 +-
 .../multivalue/MvSliceIntEvaluator.java       |  2 +-
 .../multivalue/MvSliceLongEvaluator.java      |  2 +-
 .../scalar/multivalue/MvSumIntEvaluator.java  |  2 +-
 .../scalar/multivalue/MvSumLongEvaluator.java |  2 +-
 .../MvSumUnsignedLongEvaluator.java           |  2 +-
 .../scalar/multivalue/MvZipEvaluator.java     |  2 +-
 ...ianPointDocValuesAndConstantEvaluator.java |  2 +-
 ...esianPointDocValuesAndSourceEvaluator.java |  2 +-
 ...nsCartesianSourceAndConstantEvaluator.java |  2 +-
 ...ainsCartesianSourceAndSourceEvaluator.java |  2 +-
 ...GeoPointDocValuesAndConstantEvaluator.java |  2 +-
 ...nsGeoPointDocValuesAndSourceEvaluator.java |  2 +-
 ...ContainsGeoSourceAndConstantEvaluator.java |  2 +-
 ...alContainsGeoSourceAndSourceEvaluator.java |  2 +-
 ...ianPointDocValuesAndConstantEvaluator.java |  2 +-
 ...esianPointDocValuesAndSourceEvaluator.java |  2 +-
 ...ntCartesianSourceAndConstantEvaluator.java |  2 +-
 ...ointCartesianSourceAndSourceEvaluator.java |  2 +-
 ...GeoPointDocValuesAndConstantEvaluator.java |  2 +-
 ...ntGeoPointDocValuesAndSourceEvaluator.java |  2 +-
 ...DisjointGeoSourceAndConstantEvaluator.java |  2 +-
 ...alDisjointGeoSourceAndSourceEvaluator.java |  2 +-
 ...ianPointDocValuesAndConstantEvaluator.java |  2 +-
 ...esianPointDocValuesAndSourceEvaluator.java |  2 +-
 ...tsCartesianSourceAndConstantEvaluator.java |  2 +-
 ...ectsCartesianSourceAndSourceEvaluator.java |  2 +-
 ...GeoPointDocValuesAndConstantEvaluator.java |  2 +-
 ...tsGeoPointDocValuesAndSourceEvaluator.java |  2 +-
 ...tersectsGeoSourceAndConstantEvaluator.java |  2 +-
 ...IntersectsGeoSourceAndSourceEvaluator.java |  2 +-
 ...ianPointDocValuesAndConstantEvaluator.java |  2 +-
 ...esianPointDocValuesAndSourceEvaluator.java |  2 +-
 ...inCartesianSourceAndConstantEvaluator.java |  2 +-
 ...thinCartesianSourceAndSourceEvaluator.java |  2 +-
 ...GeoPointDocValuesAndConstantEvaluator.java |  2 +-
 ...inGeoPointDocValuesAndSourceEvaluator.java |  2 +-
 ...alWithinGeoSourceAndConstantEvaluator.java |  2 +-
 ...tialWithinGeoSourceAndSourceEvaluator.java |  2 +-
 .../scalar/string/AutomataMatchEvaluator.java |  2 +-
 .../scalar/string/ConcatEvaluator.java        |  2 +-
 .../scalar/string/EndsWithEvaluator.java      |  2 +-
 .../scalar/string/LTrimEvaluator.java         |  2 +-
 .../function/scalar/string/LeftEvaluator.java |  2 +-
 .../scalar/string/LengthEvaluator.java        |  2 +-
 .../scalar/string/LocateEvaluator.java        |  2 +-
 .../scalar/string/LocateNoStartEvaluator.java |  2 +-
 .../scalar/string/RTrimEvaluator.java         |  2 +-
 .../string/ReplaceConstantEvaluator.java      |  2 +-
 .../scalar/string/ReplaceEvaluator.java       |  2 +-
 .../scalar/string/RightEvaluator.java         |  2 +-
 .../string/SplitSingleByteEvaluator.java      |  2 +-
 .../scalar/string/SplitVariableEvaluator.java |  2 +-
 .../scalar/string/StartsWithEvaluator.java    |  2 +-
 .../scalar/string/SubstringEvaluator.java     |  2 +-
 .../string/SubstringNoLengthEvaluator.java    |  2 +-
 .../scalar/string/ToLowerEvaluator.java       |  2 +-
 .../scalar/string/ToUpperEvaluator.java       |  2 +-
 .../function/scalar/string/TrimEvaluator.java |  2 +-
 .../arithmetic/AddDatetimesEvaluator.java     |  2 +-
 .../arithmetic/AddDoublesEvaluator.java       |  2 +-
 .../operator/arithmetic/AddIntsEvaluator.java |  2 +-
 .../arithmetic/AddLongsEvaluator.java         |  2 +-
 .../arithmetic/AddUnsignedLongsEvaluator.java |  2 +-
 .../arithmetic/DivDoublesEvaluator.java       |  2 +-
 .../operator/arithmetic/DivIntsEvaluator.java |  2 +-
 .../arithmetic/DivLongsEvaluator.java         |  2 +-
 .../arithmetic/DivUnsignedLongsEvaluator.java |  2 +-
 .../arithmetic/ModDoublesEvaluator.java       |  2 +-
 .../operator/arithmetic/ModIntsEvaluator.java |  2 +-
 .../arithmetic/ModLongsEvaluator.java         |  2 +-
 .../arithmetic/ModUnsignedLongsEvaluator.java |  2 +-
 .../arithmetic/MulDoublesEvaluator.java       |  2 +-
 .../operator/arithmetic/MulIntsEvaluator.java |  2 +-
 .../arithmetic/MulLongsEvaluator.java         |  2 +-
 .../arithmetic/MulUnsignedLongsEvaluator.java |  2 +-
 .../arithmetic/NegDoublesEvaluator.java       |  2 +-
 .../operator/arithmetic/NegIntsEvaluator.java |  2 +-
 .../arithmetic/NegLongsEvaluator.java         |  2 +-
 .../arithmetic/SubDatetimesEvaluator.java     |  2 +-
 .../arithmetic/SubDoublesEvaluator.java       |  2 +-
 .../operator/arithmetic/SubIntsEvaluator.java |  2 +-
 .../arithmetic/SubLongsEvaluator.java         |  2 +-
 .../arithmetic/SubUnsignedLongsEvaluator.java |  2 +-
 .../comparison/EqualsBoolsEvaluator.java      |  2 +-
 .../comparison/EqualsDoublesEvaluator.java    |  2 +-
 .../comparison/EqualsGeometriesEvaluator.java |  2 +-
 .../comparison/EqualsIntsEvaluator.java       |  2 +-
 .../comparison/EqualsKeywordsEvaluator.java   |  2 +-
 .../comparison/EqualsLongsEvaluator.java      |  2 +-
 .../GreaterThanDoublesEvaluator.java          |  2 +-
 .../comparison/GreaterThanIntsEvaluator.java  |  2 +-
 .../GreaterThanKeywordsEvaluator.java         |  2 +-
 .../comparison/GreaterThanLongsEvaluator.java |  2 +-
 .../GreaterThanOrEqualDoublesEvaluator.java   |  2 +-
 .../GreaterThanOrEqualIntsEvaluator.java      |  2 +-
 .../GreaterThanOrEqualKeywordsEvaluator.java  |  2 +-
 .../GreaterThanOrEqualLongsEvaluator.java     |  2 +-
 .../InsensitiveEqualsConstantEvaluator.java   |  2 +-
 .../InsensitiveEqualsEvaluator.java           |  2 +-
 .../comparison/LessThanDoublesEvaluator.java  |  2 +-
 .../comparison/LessThanIntsEvaluator.java     |  2 +-
 .../comparison/LessThanKeywordsEvaluator.java |  2 +-
 .../comparison/LessThanLongsEvaluator.java    |  2 +-
 .../LessThanOrEqualDoublesEvaluator.java      |  2 +-
 .../LessThanOrEqualIntsEvaluator.java         |  2 +-
 .../LessThanOrEqualKeywordsEvaluator.java     |  2 +-
 .../LessThanOrEqualLongsEvaluator.java        |  2 +-
 .../comparison/NotEqualsBoolsEvaluator.java   |  2 +-
 .../comparison/NotEqualsDoublesEvaluator.java |  2 +-
 .../NotEqualsGeometriesEvaluator.java         |  2 +-
 .../comparison/NotEqualsIntsEvaluator.java    |  2 +-
 .../NotEqualsKeywordsEvaluator.java           |  2 +-
 .../comparison/NotEqualsLongsEvaluator.java   |  2 +-
 .../evaluator/mapper/EvaluatorMapper.java     | 12 +------
 .../esql/expression/function/Warnings.java    | 26 +++++++++++++++
 190 files changed, 247 insertions(+), 199 deletions(-)

diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java
index 55a81cd7aaace..2d3c59d3f8850 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java
@@ -120,11 +120,11 @@ private TypeSpec type() {
     private MethodSpec ctor() {
         MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC);
         builder.addParameter(SOURCE, "source");
-        builder.addStatement("this.warnings = new Warnings(source)");
         processFunction.args.stream().forEach(a -> a.implementCtor(builder));
 
         builder.addParameter(DRIVER_CONTEXT, "driverContext");
         builder.addStatement("this.driverContext = driverContext");
+        builder.addStatement("this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source)");
         return builder.build();
     }
 
diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/MvEvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/MvEvaluatorImplementer.java
index 52b1c2b09b629..993b8363fb35f 100644
--- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/MvEvaluatorImplementer.java
+++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/MvEvaluatorImplementer.java
@@ -166,10 +166,10 @@ private MethodSpec ctor() {
         }
         builder.addParameter(EXPRESSION_EVALUATOR, "field");
         builder.addStatement("super(driverContext, field)");
+        builder.addParameter(DRIVER_CONTEXT, "driverContext");
         if (warnExceptions.isEmpty() == false) {
-            builder.addStatement("this.warnings = new Warnings(source)");
+            builder.addStatement("this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source)");
         }
-        builder.addParameter(DRIVER_CONTEXT, "driverContext");
         return builder.build();
     }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverContext.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverContext.java
index d645a7cbe0185..843aa4aaaa881 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverContext.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverContext.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.core.Releasable;
@@ -57,11 +58,26 @@ public class DriverContext {
 
     private final AsyncActions asyncActions = new AsyncActions();
 
+    private final WarningsMode warningsMode;
+
     public DriverContext(BigArrays bigArrays, BlockFactory blockFactory) {
+        this(bigArrays, blockFactory, WarningsMode.COLLECT);
+    }
+
+    private DriverContext(BigArrays bigArrays, BlockFactory blockFactory, WarningsMode warningsMode) {
         Objects.requireNonNull(bigArrays);
         Objects.requireNonNull(blockFactory);
         this.bigArrays = bigArrays;
         this.blockFactory = blockFactory;
+        this.warningsMode = warningsMode;
+    }
+
+    public static DriverContext getLocalDriver() {
+        return new DriverContext(
+            BigArrays.NON_RECYCLING_INSTANCE,
+            // TODO maybe this should have a small fixed limit?
+            new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE)
+        );
     }
 
     public BigArrays bigArrays() {
@@ -159,6 +175,22 @@ public void removeAsyncAction() {
         asyncActions.removeInstance();
     }
 
+    /**
+     * Evaluators should use this function to decide their warning behavior.
+     * @return an appropriate {@link WarningsMode}
+     */
+    public WarningsMode warningsMode() {
+        return warningsMode;
+    }
+
+    /**
+     * Indicates the behavior Evaluators of this context should use for reporting warnings
+     */
+    public enum WarningsMode {
+        COLLECT,
+        IGNORE
+    }
+
     private static class AsyncActions {
         private final SubscribableListener<Void> completion = new SubscribableListener<>();
         private final AtomicBoolean finished = new AtomicBoolean();
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/logical/NotEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/logical/NotEvaluator.java
index 8479e2c073d82..1ca9ddcb45747 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/logical/NotEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/logical/NotEvaluator.java
@@ -30,9 +30,9 @@ public final class NotEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public NotEvaluator(Source source, EvalOperator.ExpressionEvaluator v,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.v = v;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBooleanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBooleanEvaluator.java
index e136926ea1f5d..3109687b8c57d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBooleanEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBooleanEvaluator.java
@@ -32,9 +32,9 @@ public final class GreatestBooleanEvaluator implements EvalOperator.ExpressionEv
 
   public GreatestBooleanEvaluator(Source source, EvalOperator.ExpressionEvaluator[] values,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBytesRefEvaluator.java
index bff5eb17aa98c..0879c62ecafa6 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBytesRefEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBytesRefEvaluator.java
@@ -33,9 +33,9 @@ public final class GreatestBytesRefEvaluator implements EvalOperator.ExpressionE
 
   public GreatestBytesRefEvaluator(Source source, EvalOperator.ExpressionEvaluator[] values,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestDoubleEvaluator.java
index c82e076e6d807..0ab8c6c3588a3 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestDoubleEvaluator.java
@@ -32,9 +32,9 @@ public final class GreatestDoubleEvaluator implements EvalOperator.ExpressionEva
 
   public GreatestDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator[] values,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestIntEvaluator.java
index 28dbdd4218c1b..0969c7f708209 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestIntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestIntEvaluator.java
@@ -32,9 +32,9 @@ public final class GreatestIntEvaluator implements EvalOperator.ExpressionEvalua
 
   public GreatestIntEvaluator(Source source, EvalOperator.ExpressionEvaluator[] values,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestLongEvaluator.java
index 8e907c20742ed..3bcafa572c9e1 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestLongEvaluator.java
@@ -32,9 +32,9 @@ public final class GreatestLongEvaluator implements EvalOperator.ExpressionEvalu
 
   public GreatestLongEvaluator(Source source, EvalOperator.ExpressionEvaluator[] values,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBooleanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBooleanEvaluator.java
index f99f385ba616b..fe46511f70a90 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBooleanEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBooleanEvaluator.java
@@ -32,9 +32,9 @@ public final class LeastBooleanEvaluator implements EvalOperator.ExpressionEvalu
 
   public LeastBooleanEvaluator(Source source, EvalOperator.ExpressionEvaluator[] values,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBytesRefEvaluator.java
index 23d03102eed5e..c701da21de514 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBytesRefEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBytesRefEvaluator.java
@@ -33,9 +33,9 @@ public final class LeastBytesRefEvaluator implements EvalOperator.ExpressionEval
 
   public LeastBytesRefEvaluator(Source source, EvalOperator.ExpressionEvaluator[] values,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastDoubleEvaluator.java
index 4dae6fe78041d..f174cbc3561ad 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastDoubleEvaluator.java
@@ -32,9 +32,9 @@ public final class LeastDoubleEvaluator implements EvalOperator.ExpressionEvalua
 
   public LeastDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator[] values,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastIntEvaluator.java
index a6a1cac60146f..888e84ce7bead 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastIntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastIntEvaluator.java
@@ -32,9 +32,9 @@ public final class LeastIntEvaluator implements EvalOperator.ExpressionEvaluator
 
   public LeastIntEvaluator(Source source, EvalOperator.ExpressionEvaluator[] values,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastLongEvaluator.java
index e8416095d46b2..f270cad207381 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastLongEvaluator.java
@@ -32,9 +32,9 @@ public final class LeastLongEvaluator implements EvalOperator.ExpressionEvaluato
 
   public LeastLongEvaluator(Source source, EvalOperator.ExpressionEvaluator[] values,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Evaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Evaluator.java
index fff0acc5009f2..f4704dc7c7e27 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Evaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Evaluator.java
@@ -35,10 +35,10 @@ public final class FromBase64Evaluator implements EvalOperator.ExpressionEvaluat
 
   public FromBase64Evaluator(Source source, EvalOperator.ExpressionEvaluator field,
       BytesRefBuilder oScratch, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field = field;
     this.oScratch = oScratch;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Evaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Evaluator.java
index 2bf0df5caeecd..eb0c483c7485d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Evaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Evaluator.java
@@ -36,10 +36,10 @@ public final class ToBase64Evaluator implements EvalOperator.ExpressionEvaluator
 
   public ToBase64Evaluator(Source source, EvalOperator.ExpressionEvaluator field,
       BytesRefBuilder oScratch, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field = field;
     this.oScratch = oScratch;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffConstantEvaluator.java
index 6543e1e9bcb29..fe54f8f5f9e12 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffConstantEvaluator.java
@@ -37,11 +37,11 @@ public final class DateDiffConstantEvaluator implements EvalOperator.ExpressionE
   public DateDiffConstantEvaluator(Source source, DateDiff.Part datePartFieldUnit,
       EvalOperator.ExpressionEvaluator startTimestamp,
       EvalOperator.ExpressionEvaluator endTimestamp, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.datePartFieldUnit = datePartFieldUnit;
     this.startTimestamp = startTimestamp;
     this.endTimestamp = endTimestamp;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffEvaluator.java
index 7d7874c1beb99..dbb13c2d422dd 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffEvaluator.java
@@ -40,11 +40,11 @@ public final class DateDiffEvaluator implements EvalOperator.ExpressionEvaluator
   public DateDiffEvaluator(Source source, EvalOperator.ExpressionEvaluator unit,
       EvalOperator.ExpressionEvaluator startTimestamp,
       EvalOperator.ExpressionEvaluator endTimestamp, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.unit = unit;
     this.startTimestamp = startTimestamp;
     this.endTimestamp = endTimestamp;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java
index 4f63a5579474e..6b7c794810f19 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java
@@ -36,11 +36,11 @@ public final class DateExtractConstantEvaluator implements EvalOperator.Expressi
 
   public DateExtractConstantEvaluator(Source source, EvalOperator.ExpressionEvaluator value,
       ChronoField chronoField, ZoneId zone, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.value = value;
     this.chronoField = chronoField;
     this.zone = zone;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java
index dafe99859f77a..e2c77cd2718c4 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java
@@ -38,11 +38,11 @@ public final class DateExtractEvaluator implements EvalOperator.ExpressionEvalua
 
   public DateExtractEvaluator(Source source, EvalOperator.ExpressionEvaluator value,
       EvalOperator.ExpressionEvaluator chronoField, ZoneId zone, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.value = value;
     this.chronoField = chronoField;
     this.zone = zone;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java
index 55170fc19d05b..770230e3a5a71 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java
@@ -35,10 +35,10 @@ public final class DateFormatConstantEvaluator implements EvalOperator.Expressio
 
   public DateFormatConstantEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DateFormatter formatter, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.formatter = formatter;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java
index 9b2df06b61ceb..0ac3f5c327169 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java
@@ -38,11 +38,11 @@ public final class DateFormatEvaluator implements EvalOperator.ExpressionEvaluat
 
   public DateFormatEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       EvalOperator.ExpressionEvaluator formatter, Locale locale, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.formatter = formatter;
     this.locale = locale;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java
index 28ef039cf72c6..c08c1a54f90ba 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java
@@ -35,10 +35,10 @@ public final class DateParseConstantEvaluator implements EvalOperator.Expression
 
   public DateParseConstantEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DateFormatter formatter, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.formatter = formatter;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java
index 4f1018b446587..a28a3feb1c9b6 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java
@@ -37,11 +37,11 @@ public final class DateParseEvaluator implements EvalOperator.ExpressionEvaluato
 
   public DateParseEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       EvalOperator.ExpressionEvaluator formatter, ZoneId zoneId, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.formatter = formatter;
     this.zoneId = zoneId;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java
index 5630f963eac2b..7c027fc1143d1 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java
@@ -33,10 +33,10 @@ public final class DateTruncEvaluator implements EvalOperator.ExpressionEvaluato
 
   public DateTruncEvaluator(Source source, EvalOperator.ExpressionEvaluator fieldVal,
       Rounding.Prepared rounding, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.fieldVal = fieldVal;
     this.rounding = rounding;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java
index 51e00309f4450..ef3f08b3d2b70 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java
@@ -26,9 +26,9 @@ public final class NowEvaluator implements EvalOperator.ExpressionEvaluator {
   private final DriverContext driverContext;
 
   public NowEvaluator(Source source, long now, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.now = now;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchEvaluator.java
index 0f91568f36430..070d0a5a07edf 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchEvaluator.java
@@ -37,10 +37,10 @@ public final class CIDRMatchEvaluator implements EvalOperator.ExpressionEvaluato
 
   public CIDRMatchEvaluator(Source source, EvalOperator.ExpressionEvaluator ip,
       EvalOperator.ExpressionEvaluator[] cidrs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.ip = ip;
     this.cidrs = cidrs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsDoubleEvaluator.java
index 917f2efac561d..235182f5e6ecf 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsDoubleEvaluator.java
@@ -30,9 +30,9 @@ public final class AbsDoubleEvaluator implements EvalOperator.ExpressionEvaluato
 
   public AbsDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator fieldVal,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.fieldVal = fieldVal;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsIntEvaluator.java
index fa8a17266cbca..bec60e7bc870e 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsIntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsIntEvaluator.java
@@ -30,9 +30,9 @@ public final class AbsIntEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public AbsIntEvaluator(Source source, EvalOperator.ExpressionEvaluator fieldVal,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.fieldVal = fieldVal;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsLongEvaluator.java
index 0a8f122176f05..46f2f18c07ec7 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsLongEvaluator.java
@@ -30,9 +30,9 @@ public final class AbsLongEvaluator implements EvalOperator.ExpressionEvaluator
 
   public AbsLongEvaluator(Source source, EvalOperator.ExpressionEvaluator fieldVal,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.fieldVal = fieldVal;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosEvaluator.java
index 1d7d18ff1e437..840483e754b43 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosEvaluator.java
@@ -31,9 +31,9 @@ public final class AcosEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public AcosEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinEvaluator.java
index c5d116c3bce86..ed78b6fa29733 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinEvaluator.java
@@ -31,9 +31,9 @@ public final class AsinEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public AsinEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Evaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Evaluator.java
index c71612a98d7eb..e5abbfc6c7c74 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Evaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Evaluator.java
@@ -32,10 +32,10 @@ public final class Atan2Evaluator implements EvalOperator.ExpressionEvaluator {
 
   public Atan2Evaluator(Source source, EvalOperator.ExpressionEvaluator y,
       EvalOperator.ExpressionEvaluator x, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.y = y;
     this.x = x;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanEvaluator.java
index fd6bbfd2b8a30..364e94bd20f71 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanEvaluator.java
@@ -30,9 +30,9 @@ public final class AtanEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public AtanEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToDoubleEvaluator.java
index ccd453db17dfb..8e05a9f01fc4c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToDoubleEvaluator.java
@@ -32,9 +32,9 @@ public final class CastIntToDoubleEvaluator implements EvalOperator.ExpressionEv
 
   public CastIntToDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator v,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.v = v;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToLongEvaluator.java
index b2f965d9862e5..7dc73231f1090 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToLongEvaluator.java
@@ -32,9 +32,9 @@ public final class CastIntToLongEvaluator implements EvalOperator.ExpressionEval
 
   public CastIntToLongEvaluator(Source source, EvalOperator.ExpressionEvaluator v,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.v = v;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToUnsignedLongEvaluator.java
index 723acd3818867..8a98dd61c6b83 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToUnsignedLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToUnsignedLongEvaluator.java
@@ -32,9 +32,9 @@ public final class CastIntToUnsignedLongEvaluator implements EvalOperator.Expres
 
   public CastIntToUnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluator v,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.v = v;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToDoubleEvaluator.java
index 3a18e2f3d6c78..230ed8cbdd538 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToDoubleEvaluator.java
@@ -32,9 +32,9 @@ public final class CastLongToDoubleEvaluator implements EvalOperator.ExpressionE
 
   public CastLongToDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator v,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.v = v;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToUnsignedLongEvaluator.java
index 4b4b30ff5452d..050205ceede7b 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToUnsignedLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToUnsignedLongEvaluator.java
@@ -30,9 +30,9 @@ public final class CastLongToUnsignedLongEvaluator implements EvalOperator.Expre
 
   public CastLongToUnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluator v,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.v = v;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastUnsignedLongToDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastUnsignedLongToDoubleEvaluator.java
index f5cf77cde426d..7d76e27e426f0 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastUnsignedLongToDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastUnsignedLongToDoubleEvaluator.java
@@ -32,9 +32,9 @@ public final class CastUnsignedLongToDoubleEvaluator implements EvalOperator.Exp
 
   public CastUnsignedLongToDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator v,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.v = v;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtDoubleEvaluator.java
index adb850f6dc57d..e34ea2a314b1c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtDoubleEvaluator.java
@@ -31,9 +31,9 @@ public final class CbrtDoubleEvaluator implements EvalOperator.ExpressionEvaluat
 
   public CbrtDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtIntEvaluator.java
index 8d1fb71f79a06..fb815f9f01e33 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtIntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtIntEvaluator.java
@@ -32,9 +32,9 @@ public final class CbrtIntEvaluator implements EvalOperator.ExpressionEvaluator
 
   public CbrtIntEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtLongEvaluator.java
index 766fe75bf8288..56a7af30bcfd0 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtLongEvaluator.java
@@ -32,9 +32,9 @@ public final class CbrtLongEvaluator implements EvalOperator.ExpressionEvaluator
 
   public CbrtLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtUnsignedLongEvaluator.java
index b3b0567a058e8..eb833043c4d6f 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtUnsignedLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtUnsignedLongEvaluator.java
@@ -32,9 +32,9 @@ public final class CbrtUnsignedLongEvaluator implements EvalOperator.ExpressionE
 
   public CbrtUnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilDoubleEvaluator.java
index 30882e1c0777e..4ab5481af167e 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilDoubleEvaluator.java
@@ -30,9 +30,9 @@ public final class CeilDoubleEvaluator implements EvalOperator.ExpressionEvaluat
 
   public CeilDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosEvaluator.java
index d92c19a9637f0..e78dcbbfbb1c0 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosEvaluator.java
@@ -30,9 +30,9 @@ public final class CosEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public CosEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshEvaluator.java
index 1c113bd9655ae..211d801b75fd8 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshEvaluator.java
@@ -31,9 +31,9 @@ public final class CoshEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public CoshEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorDoubleEvaluator.java
index 867c43951818a..0ae93dddeb5a2 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorDoubleEvaluator.java
@@ -30,9 +30,9 @@ public final class FloorDoubleEvaluator implements EvalOperator.ExpressionEvalua
 
   public FloorDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java
index 8fa1e7218ac61..0a4d7a3ad6d2c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java
@@ -31,9 +31,9 @@ public final class Log10DoubleEvaluator implements EvalOperator.ExpressionEvalua
 
   public Log10DoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java
index 417074d0a47b2..147e2052af998 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java
@@ -32,9 +32,9 @@ public final class Log10IntEvaluator implements EvalOperator.ExpressionEvaluator
 
   public Log10IntEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java
index 1134de7f76b0b..565f286dcc8cd 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java
@@ -32,9 +32,9 @@ public final class Log10LongEvaluator implements EvalOperator.ExpressionEvaluato
 
   public Log10LongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java
index 6f5f036e29a62..a900585fb6ef2 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java
@@ -32,9 +32,9 @@ public final class Log10UnsignedLongEvaluator implements EvalOperator.Expression
 
   public Log10UnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogConstantEvaluator.java
index 82d34296a64d9..3688e989a45d4 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogConstantEvaluator.java
@@ -31,9 +31,9 @@ public final class LogConstantEvaluator implements EvalOperator.ExpressionEvalua
 
   public LogConstantEvaluator(Source source, EvalOperator.ExpressionEvaluator value,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.value = value;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogEvaluator.java
index 9ef5cf8d9580c..c2ccd4d64ec81 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogEvaluator.java
@@ -33,10 +33,10 @@ public final class LogEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public LogEvaluator(Source source, EvalOperator.ExpressionEvaluator base,
       EvalOperator.ExpressionEvaluator value, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.base = base;
     this.value = value;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowEvaluator.java
index 2721150e49009..7d85b12e50085 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowEvaluator.java
@@ -33,10 +33,10 @@ public final class PowEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public PowEvaluator(Source source, EvalOperator.ExpressionEvaluator base,
       EvalOperator.ExpressionEvaluator exponent, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.base = base;
     this.exponent = exponent;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleEvaluator.java
index 07d165f9b102c..40ac314983dbc 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleEvaluator.java
@@ -34,10 +34,10 @@ public final class RoundDoubleEvaluator implements EvalOperator.ExpressionEvalua
 
   public RoundDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       EvalOperator.ExpressionEvaluator decimals, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.decimals = decimals;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleNoDecimalsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleNoDecimalsEvaluator.java
index cf7300ce8aea8..b6476e867fdf4 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleNoDecimalsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleNoDecimalsEvaluator.java
@@ -30,9 +30,9 @@ public final class RoundDoubleNoDecimalsEvaluator implements EvalOperator.Expres
 
   public RoundDoubleNoDecimalsEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundIntEvaluator.java
index 0ea8111d392c5..cac749b070f62 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundIntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundIntEvaluator.java
@@ -34,10 +34,10 @@ public final class RoundIntEvaluator implements EvalOperator.ExpressionEvaluator
 
   public RoundIntEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       EvalOperator.ExpressionEvaluator decimals, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.decimals = decimals;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundLongEvaluator.java
index 9f862a4e7289b..86c44ed9fef5c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundLongEvaluator.java
@@ -32,10 +32,10 @@ public final class RoundLongEvaluator implements EvalOperator.ExpressionEvaluato
 
   public RoundLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       EvalOperator.ExpressionEvaluator decimals, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.decimals = decimals;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundUnsignedLongEvaluator.java
index 3d07675692f8d..731bfd8f59896 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundUnsignedLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundUnsignedLongEvaluator.java
@@ -32,10 +32,10 @@ public final class RoundUnsignedLongEvaluator implements EvalOperator.Expression
 
   public RoundUnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       EvalOperator.ExpressionEvaluator decimals, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.decimals = decimals;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumDoubleEvaluator.java
index 4fbac4ce96392..cb7add4a67271 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumDoubleEvaluator.java
@@ -30,9 +30,9 @@ public final class SignumDoubleEvaluator implements EvalOperator.ExpressionEvalu
 
   public SignumDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumIntEvaluator.java
index 122779e1d5e65..3e6f0e52347cb 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumIntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumIntEvaluator.java
@@ -32,9 +32,9 @@ public final class SignumIntEvaluator implements EvalOperator.ExpressionEvaluato
 
   public SignumIntEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumLongEvaluator.java
index aaae4eb6ba1ca..3b05ca81d787b 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumLongEvaluator.java
@@ -32,9 +32,9 @@ public final class SignumLongEvaluator implements EvalOperator.ExpressionEvaluat
 
   public SignumLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumUnsignedLongEvaluator.java
index 4070836e89c19..c080d5c9d1c93 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumUnsignedLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumUnsignedLongEvaluator.java
@@ -32,9 +32,9 @@ public final class SignumUnsignedLongEvaluator implements EvalOperator.Expressio
 
   public SignumUnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinEvaluator.java
index 7e7484b3f47c3..50f10aaefa99a 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinEvaluator.java
@@ -30,9 +30,9 @@ public final class SinEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public SinEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhEvaluator.java
index 6850b2e151950..1aecf68eec110 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhEvaluator.java
@@ -31,9 +31,9 @@ public final class SinhEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public SinhEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtDoubleEvaluator.java
index 60d735cdf6a2a..cb1c5da5b0cf0 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtDoubleEvaluator.java
@@ -31,9 +31,9 @@ public final class SqrtDoubleEvaluator implements EvalOperator.ExpressionEvaluat
 
   public SqrtDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtIntEvaluator.java
index 0e13726e3a4bc..26fa9f39e7059 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtIntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtIntEvaluator.java
@@ -32,9 +32,9 @@ public final class SqrtIntEvaluator implements EvalOperator.ExpressionEvaluator
 
   public SqrtIntEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtLongEvaluator.java
index 5695b23eba72b..7d306d76cd791 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtLongEvaluator.java
@@ -32,9 +32,9 @@ public final class SqrtLongEvaluator implements EvalOperator.ExpressionEvaluator
 
   public SqrtLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtUnsignedLongEvaluator.java
index fd60cbb419413..868342a59fbe4 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtUnsignedLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtUnsignedLongEvaluator.java
@@ -32,9 +32,9 @@ public final class SqrtUnsignedLongEvaluator implements EvalOperator.ExpressionE
 
   public SqrtUnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanEvaluator.java
index 78214b1c0d320..9a4d71d8b8c57 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanEvaluator.java
@@ -30,9 +30,9 @@ public final class TanEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public TanEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhEvaluator.java
index 1acd2b47174cf..5e4046d0a8c4d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhEvaluator.java
@@ -30,9 +30,9 @@ public final class TanhEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public TanhEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceBooleanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceBooleanEvaluator.java
index d63da310b7acd..294cdb4373c86 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceBooleanEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceBooleanEvaluator.java
@@ -36,11 +36,11 @@ public final class MvSliceBooleanEvaluator implements EvalOperator.ExpressionEva
   public MvSliceBooleanEvaluator(Source source, EvalOperator.ExpressionEvaluator field,
       EvalOperator.ExpressionEvaluator start, EvalOperator.ExpressionEvaluator end,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field = field;
     this.start = start;
     this.end = end;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceBytesRefEvaluator.java
index f9ad5518e5792..ce17962a59391 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceBytesRefEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceBytesRefEvaluator.java
@@ -36,11 +36,11 @@ public final class MvSliceBytesRefEvaluator implements EvalOperator.ExpressionEv
   public MvSliceBytesRefEvaluator(Source source, EvalOperator.ExpressionEvaluator field,
       EvalOperator.ExpressionEvaluator start, EvalOperator.ExpressionEvaluator end,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field = field;
     this.start = start;
     this.end = end;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceDoubleEvaluator.java
index 13eed4711c8fc..30cd51f2b23c0 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceDoubleEvaluator.java
@@ -36,11 +36,11 @@ public final class MvSliceDoubleEvaluator implements EvalOperator.ExpressionEval
   public MvSliceDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator field,
       EvalOperator.ExpressionEvaluator start, EvalOperator.ExpressionEvaluator end,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field = field;
     this.start = start;
     this.end = end;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceIntEvaluator.java
index 114677f070261..cf8e9babc88bd 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceIntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceIntEvaluator.java
@@ -35,11 +35,11 @@ public final class MvSliceIntEvaluator implements EvalOperator.ExpressionEvaluat
   public MvSliceIntEvaluator(Source source, EvalOperator.ExpressionEvaluator field,
       EvalOperator.ExpressionEvaluator start, EvalOperator.ExpressionEvaluator end,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field = field;
     this.start = start;
     this.end = end;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceLongEvaluator.java
index 207d342a1b720..e7e2b7f643db3 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceLongEvaluator.java
@@ -36,11 +36,11 @@ public final class MvSliceLongEvaluator implements EvalOperator.ExpressionEvalua
   public MvSliceLongEvaluator(Source source, EvalOperator.ExpressionEvaluator field,
       EvalOperator.ExpressionEvaluator start, EvalOperator.ExpressionEvaluator end,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field = field;
     this.start = start;
     this.end = end;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumIntEvaluator.java
index dea92249e7e0e..a61b8f71a04a0 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumIntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumIntEvaluator.java
@@ -24,7 +24,7 @@ public final class MvSumIntEvaluator extends AbstractMultivalueFunction.Abstract
   public MvSumIntEvaluator(Source source, EvalOperator.ExpressionEvaluator field,
       DriverContext driverContext) {
     super(driverContext, field);
-    this.warnings = new Warnings(source);
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumLongEvaluator.java
index dda5638fbeef5..6d37a1e7780ef 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumLongEvaluator.java
@@ -24,7 +24,7 @@ public final class MvSumLongEvaluator extends AbstractMultivalueFunction.Abstrac
   public MvSumLongEvaluator(Source source, EvalOperator.ExpressionEvaluator field,
       DriverContext driverContext) {
     super(driverContext, field);
-    this.warnings = new Warnings(source);
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumUnsignedLongEvaluator.java
index 83ca390fdd90b..9db3b2a2afde9 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumUnsignedLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumUnsignedLongEvaluator.java
@@ -24,7 +24,7 @@ public final class MvSumUnsignedLongEvaluator extends AbstractMultivalueFunction
   public MvSumUnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluator field,
       DriverContext driverContext) {
     super(driverContext, field);
-    this.warnings = new Warnings(source);
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipEvaluator.java
index 37f8804082937..e307400f98696 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipEvaluator.java
@@ -35,11 +35,11 @@ public final class MvZipEvaluator implements EvalOperator.ExpressionEvaluator {
   public MvZipEvaluator(Source source, EvalOperator.ExpressionEvaluator leftField,
       EvalOperator.ExpressionEvaluator rightField, EvalOperator.ExpressionEvaluator delim,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftField = leftField;
     this.rightField = rightField;
     this.delim = delim;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndConstantEvaluator.java
index 32bc1acf0dcd9..b5a5634bc0fd1 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndConstantEvaluator.java
@@ -35,10 +35,10 @@ public final class SpatialContainsCartesianPointDocValuesAndConstantEvaluator im
   public SpatialContainsCartesianPointDocValuesAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D[] rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndSourceEvaluator.java
index 832e8f990720c..b5e353d641002 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianPointDocValuesAndSourceEvaluator.java
@@ -38,10 +38,10 @@ public final class SpatialContainsCartesianPointDocValuesAndSourceEvaluator impl
   public SpatialContainsCartesianPointDocValuesAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndConstantEvaluator.java
index a1c8d482ddca4..3e2de0ebd397a 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndConstantEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialContainsCartesianSourceAndConstantEvaluator implements
   public SpatialContainsCartesianSourceAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D[] rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndSourceEvaluator.java
index 38c1087cc7606..9d19429519a6e 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsCartesianSourceAndSourceEvaluator.java
@@ -36,10 +36,10 @@ public final class SpatialContainsCartesianSourceAndSourceEvaluator implements E
   public SpatialContainsCartesianSourceAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndConstantEvaluator.java
index 49c7d1433dae0..f345c135747e7 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndConstantEvaluator.java
@@ -35,10 +35,10 @@ public final class SpatialContainsGeoPointDocValuesAndConstantEvaluator implemen
   public SpatialContainsGeoPointDocValuesAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D[] rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndSourceEvaluator.java
index b1fee22fcb204..bd083e382927c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoPointDocValuesAndSourceEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialContainsGeoPointDocValuesAndSourceEvaluator implements
   public SpatialContainsGeoPointDocValuesAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndConstantEvaluator.java
index 377ead4fb9387..f2316c17db1ec 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndConstantEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialContainsGeoSourceAndConstantEvaluator implements EvalO
   public SpatialContainsGeoSourceAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D[] rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndSourceEvaluator.java
index f906969a03571..7bce1a585f490 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsGeoSourceAndSourceEvaluator.java
@@ -36,10 +36,10 @@ public final class SpatialContainsGeoSourceAndSourceEvaluator implements EvalOpe
   public SpatialContainsGeoSourceAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndConstantEvaluator.java
index 1265e3eb7f496..3c46f859c80f8 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndConstantEvaluator.java
@@ -35,10 +35,10 @@ public final class SpatialDisjointCartesianPointDocValuesAndConstantEvaluator im
   public SpatialDisjointCartesianPointDocValuesAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndSourceEvaluator.java
index 6a6f1a292a598..96c72ee137804 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianPointDocValuesAndSourceEvaluator.java
@@ -38,10 +38,10 @@ public final class SpatialDisjointCartesianPointDocValuesAndSourceEvaluator impl
   public SpatialDisjointCartesianPointDocValuesAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndConstantEvaluator.java
index 46dded83d3af2..6e5becc402135 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndConstantEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialDisjointCartesianSourceAndConstantEvaluator implements
   public SpatialDisjointCartesianSourceAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndSourceEvaluator.java
index a7cea9330aa00..5c888eddcac68 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointCartesianSourceAndSourceEvaluator.java
@@ -36,10 +36,10 @@ public final class SpatialDisjointCartesianSourceAndSourceEvaluator implements E
   public SpatialDisjointCartesianSourceAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndConstantEvaluator.java
index e52c04c4d72b4..6fa2d7a6dd639 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndConstantEvaluator.java
@@ -35,10 +35,10 @@ public final class SpatialDisjointGeoPointDocValuesAndConstantEvaluator implemen
   public SpatialDisjointGeoPointDocValuesAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndSourceEvaluator.java
index 3e8df2f0b7852..96a2a06dbe241 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoPointDocValuesAndSourceEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialDisjointGeoPointDocValuesAndSourceEvaluator implements
   public SpatialDisjointGeoPointDocValuesAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndConstantEvaluator.java
index c061ecf09ced8..c3930168ae594 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndConstantEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialDisjointGeoSourceAndConstantEvaluator implements EvalO
   public SpatialDisjointGeoSourceAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndSourceEvaluator.java
index 9d617a39973f2..467d2518d9a9c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointGeoSourceAndSourceEvaluator.java
@@ -36,10 +36,10 @@ public final class SpatialDisjointGeoSourceAndSourceEvaluator implements EvalOpe
   public SpatialDisjointGeoSourceAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndConstantEvaluator.java
index 775680d2cde42..56912e3233a4c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndConstantEvaluator.java
@@ -35,10 +35,10 @@ public final class SpatialIntersectsCartesianPointDocValuesAndConstantEvaluator
   public SpatialIntersectsCartesianPointDocValuesAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndSourceEvaluator.java
index 4b6d8168000ae..0d4a5aa196f4b 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianPointDocValuesAndSourceEvaluator.java
@@ -38,10 +38,10 @@ public final class SpatialIntersectsCartesianPointDocValuesAndSourceEvaluator im
   public SpatialIntersectsCartesianPointDocValuesAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndConstantEvaluator.java
index d7e6507bec3e0..26c4abdc51ecf 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndConstantEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialIntersectsCartesianSourceAndConstantEvaluator implemen
   public SpatialIntersectsCartesianSourceAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndSourceEvaluator.java
index f1581fefbf0f7..8ad15d9f5a881 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsCartesianSourceAndSourceEvaluator.java
@@ -36,10 +36,10 @@ public final class SpatialIntersectsCartesianSourceAndSourceEvaluator implements
   public SpatialIntersectsCartesianSourceAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndConstantEvaluator.java
index c1618b407b0e5..405d013a77f5a 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndConstantEvaluator.java
@@ -35,10 +35,10 @@ public final class SpatialIntersectsGeoPointDocValuesAndConstantEvaluator implem
   public SpatialIntersectsGeoPointDocValuesAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndSourceEvaluator.java
index 89ba6c79c06d9..f385ee992096c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoPointDocValuesAndSourceEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialIntersectsGeoPointDocValuesAndSourceEvaluator implemen
   public SpatialIntersectsGeoPointDocValuesAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndConstantEvaluator.java
index 8fa0f02b783bd..dea6989a830ab 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndConstantEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialIntersectsGeoSourceAndConstantEvaluator implements Eva
   public SpatialIntersectsGeoSourceAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndSourceEvaluator.java
index a6dc48b761989..2b5dd689ca3a9 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsGeoSourceAndSourceEvaluator.java
@@ -36,10 +36,10 @@ public final class SpatialIntersectsGeoSourceAndSourceEvaluator implements EvalO
   public SpatialIntersectsGeoSourceAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndConstantEvaluator.java
index f17a5183e9a33..5c31df936236b 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndConstantEvaluator.java
@@ -35,10 +35,10 @@ public final class SpatialWithinCartesianPointDocValuesAndConstantEvaluator impl
   public SpatialWithinCartesianPointDocValuesAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndSourceEvaluator.java
index 702f599700691..1a6e8a1cd1721 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianPointDocValuesAndSourceEvaluator.java
@@ -38,10 +38,10 @@ public final class SpatialWithinCartesianPointDocValuesAndSourceEvaluator implem
   public SpatialWithinCartesianPointDocValuesAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndConstantEvaluator.java
index 2e16e18b5677e..1a1bd4cd64535 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndConstantEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialWithinCartesianSourceAndConstantEvaluator implements E
   public SpatialWithinCartesianSourceAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndSourceEvaluator.java
index 1714ff54543ec..43440a165de74 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinCartesianSourceAndSourceEvaluator.java
@@ -36,10 +36,10 @@ public final class SpatialWithinCartesianSourceAndSourceEvaluator implements Eva
   public SpatialWithinCartesianSourceAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndConstantEvaluator.java
index 2df0772da62b3..d19182ffb2341 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndConstantEvaluator.java
@@ -35,10 +35,10 @@ public final class SpatialWithinGeoPointDocValuesAndConstantEvaluator implements
   public SpatialWithinGeoPointDocValuesAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndSourceEvaluator.java
index 458532018ffdf..761e0f2019cec 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoPointDocValuesAndSourceEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialWithinGeoPointDocValuesAndSourceEvaluator implements E
   public SpatialWithinGeoPointDocValuesAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndConstantEvaluator.java
index f284cc124260b..cca5ef92918d8 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndConstantEvaluator.java
@@ -37,10 +37,10 @@ public final class SpatialWithinGeoSourceAndConstantEvaluator implements EvalOpe
   public SpatialWithinGeoSourceAndConstantEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, Component2D rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndSourceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndSourceEvaluator.java
index 2f18cb11d06ed..bbeb07f6eefc5 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndSourceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinGeoSourceAndSourceEvaluator.java
@@ -36,10 +36,10 @@ public final class SpatialWithinGeoSourceAndSourceEvaluator implements EvalOpera
   public SpatialWithinGeoSourceAndSourceEvaluator(Source source,
       EvalOperator.ExpressionEvaluator leftValue, EvalOperator.ExpressionEvaluator rightValue,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.leftValue = leftValue;
     this.rightValue = rightValue;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatchEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatchEvaluator.java
index 223f607a8b23c..e8859b70e6586 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatchEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatchEvaluator.java
@@ -38,11 +38,11 @@ public final class AutomataMatchEvaluator implements EvalOperator.ExpressionEval
 
   public AutomataMatchEvaluator(Source source, EvalOperator.ExpressionEvaluator input,
       ByteRunAutomaton automaton, String pattern, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.input = input;
     this.automaton = automaton;
     this.pattern = pattern;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java
index 0b539a5fdaf7e..e73cc58590fc1 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java
@@ -37,10 +37,10 @@ public final class ConcatEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public ConcatEvaluator(Source source, BreakingBytesRefBuilder scratch,
       EvalOperator.ExpressionEvaluator[] values, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.scratch = scratch;
     this.values = values;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java
index 7c00bd2e27cce..a644b097d18f9 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java
@@ -35,10 +35,10 @@ public final class EndsWithEvaluator implements EvalOperator.ExpressionEvaluator
 
   public EndsWithEvaluator(Source source, EvalOperator.ExpressionEvaluator str,
       EvalOperator.ExpressionEvaluator suffix, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.str = str;
     this.suffix = suffix;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java
index f04e56d0b87f3..05c7c425ed9ab 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java
@@ -31,9 +31,9 @@ public final class LTrimEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public LTrimEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java
index 6c0e392cf6120..8effe5e6d72cd 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java
@@ -42,12 +42,12 @@ public final class LeftEvaluator implements EvalOperator.ExpressionEvaluator {
   public LeftEvaluator(Source source, BytesRef out, UnicodeUtil.UTF8CodePoint cp,
       EvalOperator.ExpressionEvaluator str, EvalOperator.ExpressionEvaluator length,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.out = out;
     this.cp = cp;
     this.str = str;
     this.length = length;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthEvaluator.java
index 25877cfc3d5be..d45b28e7b7a36 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthEvaluator.java
@@ -33,9 +33,9 @@ public final class LengthEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public LengthEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateEvaluator.java
index e62d32dca1ef5..82cbd88f0db09 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateEvaluator.java
@@ -38,11 +38,11 @@ public final class LocateEvaluator implements EvalOperator.ExpressionEvaluator {
   public LocateEvaluator(Source source, EvalOperator.ExpressionEvaluator str,
       EvalOperator.ExpressionEvaluator substr, EvalOperator.ExpressionEvaluator start,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.str = str;
     this.substr = substr;
     this.start = start;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateNoStartEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateNoStartEvaluator.java
index 4fcffcb88412e..a6c9fc92ef663 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateNoStartEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateNoStartEvaluator.java
@@ -35,10 +35,10 @@ public final class LocateNoStartEvaluator implements EvalOperator.ExpressionEval
 
   public LocateNoStartEvaluator(Source source, EvalOperator.ExpressionEvaluator str,
       EvalOperator.ExpressionEvaluator substr, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.str = str;
     this.substr = substr;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java
index 8cbd454895a67..a6dcdb25f2dfc 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java
@@ -31,9 +31,9 @@ public final class RTrimEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public RTrimEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java
index 09a5429f351bc..2b898377f59f6 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java
@@ -37,11 +37,11 @@ public final class ReplaceConstantEvaluator implements EvalOperator.ExpressionEv
 
   public ReplaceConstantEvaluator(Source source, EvalOperator.ExpressionEvaluator str,
       Pattern regex, EvalOperator.ExpressionEvaluator newStr, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.str = str;
     this.regex = regex;
     this.newStr = newStr;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceEvaluator.java
index 782a0323bc82a..a6544f0b16817 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceEvaluator.java
@@ -37,11 +37,11 @@ public final class ReplaceEvaluator implements EvalOperator.ExpressionEvaluator
   public ReplaceEvaluator(Source source, EvalOperator.ExpressionEvaluator str,
       EvalOperator.ExpressionEvaluator regex, EvalOperator.ExpressionEvaluator newStr,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.str = str;
     this.regex = regex;
     this.newStr = newStr;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java
index f8d3a8dae348f..57cad6c63242d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java
@@ -42,12 +42,12 @@ public final class RightEvaluator implements EvalOperator.ExpressionEvaluator {
   public RightEvaluator(Source source, BytesRef out, UnicodeUtil.UTF8CodePoint cp,
       EvalOperator.ExpressionEvaluator str, EvalOperator.ExpressionEvaluator length,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.out = out;
     this.cp = cp;
     this.str = str;
     this.length = length;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java
index 89138d2f54f64..d58b1aee0ee9d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java
@@ -36,11 +36,11 @@ public final class SplitSingleByteEvaluator implements EvalOperator.ExpressionEv
 
   public SplitSingleByteEvaluator(Source source, EvalOperator.ExpressionEvaluator str, byte delim,
       BytesRef scratch, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.str = str;
     this.delim = delim;
     this.scratch = scratch;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java
index be072452f9e88..753febd88ca58 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java
@@ -36,11 +36,11 @@ public final class SplitVariableEvaluator implements EvalOperator.ExpressionEval
 
   public SplitVariableEvaluator(Source source, EvalOperator.ExpressionEvaluator str,
       EvalOperator.ExpressionEvaluator delim, BytesRef scratch, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.str = str;
     this.delim = delim;
     this.scratch = scratch;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithEvaluator.java
index 83527c4b1aa24..92e015642ce73 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithEvaluator.java
@@ -35,10 +35,10 @@ public final class StartsWithEvaluator implements EvalOperator.ExpressionEvaluat
 
   public StartsWithEvaluator(Source source, EvalOperator.ExpressionEvaluator str,
       EvalOperator.ExpressionEvaluator prefix, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.str = str;
     this.prefix = prefix;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringEvaluator.java
index 98fc2510a1f89..92a2d622eaf56 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringEvaluator.java
@@ -38,11 +38,11 @@ public final class SubstringEvaluator implements EvalOperator.ExpressionEvaluato
   public SubstringEvaluator(Source source, EvalOperator.ExpressionEvaluator str,
       EvalOperator.ExpressionEvaluator start, EvalOperator.ExpressionEvaluator length,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.str = str;
     this.start = start;
     this.length = length;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringNoLengthEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringNoLengthEvaluator.java
index 442ffd517d43c..9177b31ab64fd 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringNoLengthEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringNoLengthEvaluator.java
@@ -35,10 +35,10 @@ public final class SubstringNoLengthEvaluator implements EvalOperator.Expression
 
   public SubstringNoLengthEvaluator(Source source, EvalOperator.ExpressionEvaluator str,
       EvalOperator.ExpressionEvaluator start, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.str = str;
     this.start = start;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java
index ac032e60a6b4b..ee30b2b282162 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerEvaluator.java
@@ -34,10 +34,10 @@ public final class ToLowerEvaluator implements EvalOperator.ExpressionEvaluator
 
   public ToLowerEvaluator(Source source, EvalOperator.ExpressionEvaluator val, Locale locale,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.locale = locale;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java
index e79aa71a0e9d2..cf72804b7e354 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperEvaluator.java
@@ -34,10 +34,10 @@ public final class ToUpperEvaluator implements EvalOperator.ExpressionEvaluator
 
   public ToUpperEvaluator(Source source, EvalOperator.ExpressionEvaluator val, Locale locale,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.locale = locale;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimEvaluator.java
index fd357506c91a0..d4a884fc97b9c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimEvaluator.java
@@ -31,9 +31,9 @@ public final class TrimEvaluator implements EvalOperator.ExpressionEvaluator {
 
   public TrimEvaluator(Source source, EvalOperator.ExpressionEvaluator val,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.val = val;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java
index a48c658ee825b..44ed1ebebd817 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java
@@ -35,10 +35,10 @@ public final class AddDatetimesEvaluator implements EvalOperator.ExpressionEvalu
 
   public AddDatetimesEvaluator(Source source, EvalOperator.ExpressionEvaluator datetime,
       TemporalAmount temporalAmount, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.datetime = datetime;
     this.temporalAmount = temporalAmount;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java
index a9b2ebef30125..fa5b631accb77 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java
@@ -32,10 +32,10 @@ public final class AddDoublesEvaluator implements EvalOperator.ExpressionEvaluat
 
   public AddDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddIntsEvaluator.java
index aeba6794fe9dd..570b666676330 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddIntsEvaluator.java
@@ -33,10 +33,10 @@ public final class AddIntsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public AddIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddLongsEvaluator.java
index 8f5b399e39b77..71dda4fd9752e 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddLongsEvaluator.java
@@ -33,10 +33,10 @@ public final class AddLongsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public AddLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddUnsignedLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddUnsignedLongsEvaluator.java
index 36096e878ea82..16db58fe5bd6a 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddUnsignedLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddUnsignedLongsEvaluator.java
@@ -33,10 +33,10 @@ public final class AddUnsignedLongsEvaluator implements EvalOperator.ExpressionE
 
   public AddUnsignedLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivDoublesEvaluator.java
index f158cd20fc87c..20d1647d6bc99 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivDoublesEvaluator.java
@@ -33,10 +33,10 @@ public final class DivDoublesEvaluator implements EvalOperator.ExpressionEvaluat
 
   public DivDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivIntsEvaluator.java
index db4c22491be31..a1b4f3857ad0d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivIntsEvaluator.java
@@ -33,10 +33,10 @@ public final class DivIntsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public DivIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivLongsEvaluator.java
index 359f549d137ee..902d7d2f0b98c 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivLongsEvaluator.java
@@ -33,10 +33,10 @@ public final class DivLongsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public DivLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivUnsignedLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivUnsignedLongsEvaluator.java
index bf3d579788dc4..65ba269840121 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivUnsignedLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivUnsignedLongsEvaluator.java
@@ -33,10 +33,10 @@ public final class DivUnsignedLongsEvaluator implements EvalOperator.ExpressionE
 
   public DivUnsignedLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModDoublesEvaluator.java
index 659e3b2c3c89f..0f1b344e6add7 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModDoublesEvaluator.java
@@ -33,10 +33,10 @@ public final class ModDoublesEvaluator implements EvalOperator.ExpressionEvaluat
 
   public ModDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModIntsEvaluator.java
index 97916c7c0b1ff..014bc32e0e250 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModIntsEvaluator.java
@@ -33,10 +33,10 @@ public final class ModIntsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public ModIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModLongsEvaluator.java
index 89d598b5b74c4..3436c10521b64 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModLongsEvaluator.java
@@ -33,10 +33,10 @@ public final class ModLongsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public ModLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModUnsignedLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModUnsignedLongsEvaluator.java
index 11764071bb034..b031a21e79f73 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModUnsignedLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModUnsignedLongsEvaluator.java
@@ -33,10 +33,10 @@ public final class ModUnsignedLongsEvaluator implements EvalOperator.ExpressionE
 
   public ModUnsignedLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java
index 6a035b343d12f..4b27514cfda8b 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java
@@ -32,10 +32,10 @@ public final class MulDoublesEvaluator implements EvalOperator.ExpressionEvaluat
 
   public MulDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulIntsEvaluator.java
index 5c8b11fdd24b6..089765b1662c4 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulIntsEvaluator.java
@@ -33,10 +33,10 @@ public final class MulIntsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public MulIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulLongsEvaluator.java
index 87d547fc0e593..2cbc69a7face1 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulLongsEvaluator.java
@@ -33,10 +33,10 @@ public final class MulLongsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public MulLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulUnsignedLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulUnsignedLongsEvaluator.java
index 8b85459fe49c8..3a74466a9bc45 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulUnsignedLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulUnsignedLongsEvaluator.java
@@ -33,10 +33,10 @@ public final class MulUnsignedLongsEvaluator implements EvalOperator.ExpressionE
 
   public MulUnsignedLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegDoublesEvaluator.java
index 2b996c7174ceb..edd9db230cd00 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegDoublesEvaluator.java
@@ -30,9 +30,9 @@ public final class NegDoublesEvaluator implements EvalOperator.ExpressionEvaluat
 
   public NegDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator v,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.v = v;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegIntsEvaluator.java
index e3b8552b81ee6..0b27ba7f46153 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegIntsEvaluator.java
@@ -31,9 +31,9 @@ public final class NegIntsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public NegIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator v,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.v = v;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegLongsEvaluator.java
index b269d4c8c3fd9..e6c2ccb469bb8 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegLongsEvaluator.java
@@ -31,9 +31,9 @@ public final class NegLongsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public NegLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator v,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.v = v;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java
index ab162ea120004..f87f3c217e16e 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java
@@ -35,10 +35,10 @@ public final class SubDatetimesEvaluator implements EvalOperator.ExpressionEvalu
 
   public SubDatetimesEvaluator(Source source, EvalOperator.ExpressionEvaluator datetime,
       TemporalAmount temporalAmount, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.datetime = datetime;
     this.temporalAmount = temporalAmount;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java
index ff1e3d584772b..9fb0dbc0c5df6 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java
@@ -32,10 +32,10 @@ public final class SubDoublesEvaluator implements EvalOperator.ExpressionEvaluat
 
   public SubDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubIntsEvaluator.java
index cc47c615e1862..ec572bd491ec9 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubIntsEvaluator.java
@@ -33,10 +33,10 @@ public final class SubIntsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public SubIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubLongsEvaluator.java
index de3e0dd0dcbad..eaa1efeb3922d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubLongsEvaluator.java
@@ -33,10 +33,10 @@ public final class SubLongsEvaluator implements EvalOperator.ExpressionEvaluator
 
   public SubLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubUnsignedLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubUnsignedLongsEvaluator.java
index 7ab72956562de..f221e9b072351 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubUnsignedLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubUnsignedLongsEvaluator.java
@@ -33,10 +33,10 @@ public final class SubUnsignedLongsEvaluator implements EvalOperator.ExpressionE
 
   public SubUnsignedLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsBoolsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsBoolsEvaluator.java
index 185e225fadb91..10f51451752ba 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsBoolsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsBoolsEvaluator.java
@@ -32,10 +32,10 @@ public final class EqualsBoolsEvaluator implements EvalOperator.ExpressionEvalua
 
   public EqualsBoolsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsDoublesEvaluator.java
index 9e0c62d118eca..bbf78fb01aa43 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsDoublesEvaluator.java
@@ -34,10 +34,10 @@ public final class EqualsDoublesEvaluator implements EvalOperator.ExpressionEval
 
   public EqualsDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsGeometriesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsGeometriesEvaluator.java
index 192a5272789b3..ab7464e45e573 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsGeometriesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsGeometriesEvaluator.java
@@ -35,10 +35,10 @@ public final class EqualsGeometriesEvaluator implements EvalOperator.ExpressionE
 
   public EqualsGeometriesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsIntsEvaluator.java
index 5728b9be40b4a..c40087ed69012 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsIntsEvaluator.java
@@ -34,10 +34,10 @@ public final class EqualsIntsEvaluator implements EvalOperator.ExpressionEvaluat
 
   public EqualsIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsKeywordsEvaluator.java
index 2d3b679960d15..95cfd5a03de96 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsKeywordsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsKeywordsEvaluator.java
@@ -35,10 +35,10 @@ public final class EqualsKeywordsEvaluator implements EvalOperator.ExpressionEva
 
   public EqualsKeywordsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsLongsEvaluator.java
index fa4da286cde21..bf2479f50d9a6 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsLongsEvaluator.java
@@ -34,10 +34,10 @@ public final class EqualsLongsEvaluator implements EvalOperator.ExpressionEvalua
 
   public EqualsLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanDoublesEvaluator.java
index e40ce0df22a14..d962ae0715902 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanDoublesEvaluator.java
@@ -34,10 +34,10 @@ public final class GreaterThanDoublesEvaluator implements EvalOperator.Expressio
 
   public GreaterThanDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanIntsEvaluator.java
index e892430791344..a080d435d3249 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanIntsEvaluator.java
@@ -34,10 +34,10 @@ public final class GreaterThanIntsEvaluator implements EvalOperator.ExpressionEv
 
   public GreaterThanIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanKeywordsEvaluator.java
index 82f8f62097545..3f02de9493872 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanKeywordsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanKeywordsEvaluator.java
@@ -35,10 +35,10 @@ public final class GreaterThanKeywordsEvaluator implements EvalOperator.Expressi
 
   public GreaterThanKeywordsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanLongsEvaluator.java
index 0db53292de9bb..1ad7026f2c037 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanLongsEvaluator.java
@@ -34,10 +34,10 @@ public final class GreaterThanLongsEvaluator implements EvalOperator.ExpressionE
 
   public GreaterThanLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualDoublesEvaluator.java
index 2dabfdb8c0e46..a80be6b777071 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualDoublesEvaluator.java
@@ -34,10 +34,10 @@ public final class GreaterThanOrEqualDoublesEvaluator implements EvalOperator.Ex
 
   public GreaterThanOrEqualDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualIntsEvaluator.java
index 12674aee9d469..5b181e7bcea79 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualIntsEvaluator.java
@@ -34,10 +34,10 @@ public final class GreaterThanOrEqualIntsEvaluator implements EvalOperator.Expre
 
   public GreaterThanOrEqualIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualKeywordsEvaluator.java
index 433a88cd5f5b8..46db71e878e93 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualKeywordsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualKeywordsEvaluator.java
@@ -35,10 +35,10 @@ public final class GreaterThanOrEqualKeywordsEvaluator implements EvalOperator.E
 
   public GreaterThanOrEqualKeywordsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualLongsEvaluator.java
index f7b040cbcde58..1967e9b29eddf 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualLongsEvaluator.java
@@ -34,10 +34,10 @@ public final class GreaterThanOrEqualLongsEvaluator implements EvalOperator.Expr
 
   public GreaterThanOrEqualLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsConstantEvaluator.java
index e8f02f232bbf7..c431e37bd98d5 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsConstantEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsConstantEvaluator.java
@@ -36,10 +36,10 @@ public final class InsensitiveEqualsConstantEvaluator implements EvalOperator.Ex
 
   public InsensitiveEqualsConstantEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       ByteRunAutomaton rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsEvaluator.java
index a3b5a743daab8..7a0f76ff6597a 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsEvaluator.java
@@ -35,10 +35,10 @@ public final class InsensitiveEqualsEvaluator implements EvalOperator.Expression
 
   public InsensitiveEqualsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanDoublesEvaluator.java
index 4ad14555619f7..e2c11e3ce0950 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanDoublesEvaluator.java
@@ -34,10 +34,10 @@ public final class LessThanDoublesEvaluator implements EvalOperator.ExpressionEv
 
   public LessThanDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanIntsEvaluator.java
index f54a4dea0a1dc..fb8bf746f0d19 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanIntsEvaluator.java
@@ -34,10 +34,10 @@ public final class LessThanIntsEvaluator implements EvalOperator.ExpressionEvalu
 
   public LessThanIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanKeywordsEvaluator.java
index 0085bd7f37eda..b826a87d8f869 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanKeywordsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanKeywordsEvaluator.java
@@ -35,10 +35,10 @@ public final class LessThanKeywordsEvaluator implements EvalOperator.ExpressionE
 
   public LessThanKeywordsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanLongsEvaluator.java
index bf0241d05c500..f75b43e6ec7f4 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanLongsEvaluator.java
@@ -34,10 +34,10 @@ public final class LessThanLongsEvaluator implements EvalOperator.ExpressionEval
 
   public LessThanLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualDoublesEvaluator.java
index 4ff3a3fb65b14..5c03b488ff477 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualDoublesEvaluator.java
@@ -34,10 +34,10 @@ public final class LessThanOrEqualDoublesEvaluator implements EvalOperator.Expre
 
   public LessThanOrEqualDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualIntsEvaluator.java
index 00afea476db1d..6fe1996a52f29 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualIntsEvaluator.java
@@ -34,10 +34,10 @@ public final class LessThanOrEqualIntsEvaluator implements EvalOperator.Expressi
 
   public LessThanOrEqualIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualKeywordsEvaluator.java
index 460b0ed07d6da..4572929d0f6fc 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualKeywordsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualKeywordsEvaluator.java
@@ -35,10 +35,10 @@ public final class LessThanOrEqualKeywordsEvaluator implements EvalOperator.Expr
 
   public LessThanOrEqualKeywordsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualLongsEvaluator.java
index 5a293be95484d..42642205c451d 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualLongsEvaluator.java
@@ -34,10 +34,10 @@ public final class LessThanOrEqualLongsEvaluator implements EvalOperator.Express
 
   public LessThanOrEqualLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsBoolsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsBoolsEvaluator.java
index 29af62dbbc8ce..46357ab390325 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsBoolsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsBoolsEvaluator.java
@@ -32,10 +32,10 @@ public final class NotEqualsBoolsEvaluator implements EvalOperator.ExpressionEva
 
   public NotEqualsBoolsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsDoublesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsDoublesEvaluator.java
index d24ff131d21e0..0004f7bc23a31 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsDoublesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsDoublesEvaluator.java
@@ -34,10 +34,10 @@ public final class NotEqualsDoublesEvaluator implements EvalOperator.ExpressionE
 
   public NotEqualsDoublesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsGeometriesEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsGeometriesEvaluator.java
index abc0323722e23..aec5629c3b7ff 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsGeometriesEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsGeometriesEvaluator.java
@@ -35,10 +35,10 @@ public final class NotEqualsGeometriesEvaluator implements EvalOperator.Expressi
 
   public NotEqualsGeometriesEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsIntsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsIntsEvaluator.java
index 23b49df95aa66..69a7ffdb39631 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsIntsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsIntsEvaluator.java
@@ -34,10 +34,10 @@ public final class NotEqualsIntsEvaluator implements EvalOperator.ExpressionEval
 
   public NotEqualsIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsKeywordsEvaluator.java
index e4980ffeac829..6ca5d8ddc0b69 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsKeywordsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsKeywordsEvaluator.java
@@ -35,10 +35,10 @@ public final class NotEqualsKeywordsEvaluator implements EvalOperator.Expression
 
   public NotEqualsKeywordsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsLongsEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsLongsEvaluator.java
index e6dc060bf837d..a0b176952acfd 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsLongsEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsLongsEvaluator.java
@@ -34,10 +34,10 @@ public final class NotEqualsLongsEvaluator implements EvalOperator.ExpressionEva
 
   public NotEqualsLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs,
       EvalOperator.ExpressionEvaluator rhs, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.lhs = lhs;
     this.rhs = rhs;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java
index 095d2ebe2a4bd..5888e30747557 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java
@@ -7,11 +7,7 @@
 
 package org.elasticsearch.xpack.esql.evaluator.mapper;
 
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.breaker.NoopCircuitBreaker;
-import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
-import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
@@ -71,12 +67,6 @@ public Block eval(Page page) {
 
             @Override
             public void close() {}
-        }).get(
-            new DriverContext(
-                BigArrays.NON_RECYCLING_INSTANCE,
-                // TODO maybe this should have a small fixed limit?
-                new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE)
-            )
-        ).eval(new Page(1)), 0);
+        }).get(DriverContext.getLocalDriver()).eval(new Page(1)), 0);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/Warnings.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/Warnings.java
index ae1072a0a5bf8..630cf62d0030a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/Warnings.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/Warnings.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.expression.function;
 
+import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
 import static org.elasticsearch.common.logging.HeaderWarning.addWarning;
@@ -23,6 +24,31 @@ public class Warnings {
 
     private int addedWarnings;
 
+    public static final Warnings NOOP_WARNINGS = new Warnings(Source.EMPTY) {
+        @Override
+        public void registerException(Exception exception) {
+            // this space intentionally left blank
+        }
+    };
+
+    /**
+     * Create a new warnings object based on the given mode
+     * @param warningsMode The warnings collection strategy to use
+     * @param source used to indicate where in the query the warning occured
+     * @return A warnings collector object
+     */
+    public static Warnings createWarnings(DriverContext.WarningsMode warningsMode, Source source) {
+        switch (warningsMode) {
+            case COLLECT -> {
+                return new Warnings(source);
+            }
+            case IGNORE -> {
+                return NOOP_WARNINGS;
+            }
+        }
+        throw new IllegalStateException("Unreachable");
+    }
+
     public Warnings(Source source) {
         location = format("Line {}:{}: ", source.source().getLineNumber(), source.source().getColumnNumber());
         first = format(

From 4161e4d2e247c5a65cfbac84aab1813e03fa5f51 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Tue, 4 Jun 2024 16:06:19 +0200
Subject: [PATCH 130/208] Ensure synthetic source and dv codec are enabled with
 logs index mode. (#109269)

After running the elastic/logs track with logs index mode enabled, I noticed that _source was still getting stored.
The issue was that other index modes than time_series weren't propagated to Indexmetadata and IndexSettings classes. Additionally the synthetic source defaults in SourceFieldMapper were geared towards time series index mode only. This change addresses this.
---
 .../test/aggregations/time_series.yml         |  6 +--
 .../rest-api-spec/test/logsdb/10_settings.yml |  4 ++
 .../cluster/metadata/IndexMetadata.java       |  7 +--
 .../index/mapper/SourceFieldMapper.java       | 41 ++++++++++++----
 .../index/LogsIndexModeTests.java             | 31 +++++++-----
 .../index/codec/PerFieldMapperCodecTests.java | 47 +++++++++++++++----
 .../index/mapper/SourceFieldMapperTests.java  |  7 +++
 .../index/mapper/MapperServiceTestCase.java   |  5 ++
 8 files changed, 114 insertions(+), 34 deletions(-)

diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
index 7800923ab1580..1703d4908a753 100644
--- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
+++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
@@ -294,11 +294,11 @@ setup:
 ---
 "Configure with no synthetic source":
   - requires:
-      cluster_features: ["gte_v8.9.0"]
-      reason: "Error message fix in 8.9"
+      cluster_features: ["gte_v8.15.0"]
+      reason: "Error message changed in 8.15.0"
 
   - do:
-      catch: '/Time series indices only support synthetic source./'
+      catch: '/Indices with with index mode \[time_series\] only support synthetic source/'
       indices.create:
         index: tsdb_error
         body:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
index 95075da20fe5e..128903f4faac8 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
@@ -77,6 +77,10 @@ create logs index:
   - is_true: test
   - match: { test.settings.index.mode: "logs" }
 
+  - do:
+      indices.get_mapping:
+        index: test
+  - match: { test.mappings._source.mode: synthetic }
 
 ---
 using default timestamp field mapping:
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
index 681dcb3e314e3..64809c963cb6d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -2267,8 +2267,9 @@ IndexMetadata build(boolean repair) {
             }
 
             final boolean isSearchableSnapshot = SearchableSnapshotsSettings.isSearchableSnapshotStore(settings);
-            final String indexMode = settings.get(IndexSettings.MODE.getKey());
-            final boolean isTsdb = indexMode != null && IndexMode.TIME_SERIES.getName().equals(indexMode.toLowerCase(Locale.ROOT));
+            String indexModeString = settings.get(IndexSettings.MODE.getKey());
+            final IndexMode indexMode = indexModeString != null ? IndexMode.fromString(indexModeString.toLowerCase(Locale.ROOT)) : null;
+            final boolean isTsdb = indexMode == IndexMode.TIME_SERIES;
             return new IndexMetadata(
                 new Index(index, uuid),
                 version,
@@ -2308,7 +2309,7 @@ IndexMetadata build(boolean repair) {
                 AutoExpandReplicas.SETTING.get(settings),
                 isSearchableSnapshot,
                 isSearchableSnapshot && settings.getAsBoolean(SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY, false),
-                isTsdb ? IndexMode.TIME_SERIES : null,
+                indexMode,
                 isTsdb ? IndexSettings.TIME_SERIES_START_TIME.get(settings) : null,
                 isTsdb ? IndexSettings.TIME_SERIES_END_TIME.get(settings) : null,
                 SETTING_INDEX_VERSION_COMPATIBILITY.get(settings),
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index b819ffb0ef6ad..045a17409877b 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -69,6 +69,14 @@ private enum Mode {
         IndexMode.TIME_SERIES
     );
 
+    private static final SourceFieldMapper LOGS_DEFAULT = new SourceFieldMapper(
+        Mode.SYNTHETIC,
+        Explicit.IMPLICIT_TRUE,
+        Strings.EMPTY_ARRAY,
+        Strings.EMPTY_ARRAY,
+        IndexMode.LOGS
+    );
+
     /*
      * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below
      * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version.
@@ -156,7 +164,8 @@ protected Parameter<?>[] getParameters() {
 
         private boolean isDefault() {
             Mode m = mode.get();
-            if (m != null && (((indexMode == IndexMode.TIME_SERIES && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
+            if (m != null
+                && (((indexMode != null && indexMode.isSyntheticSourceEnabled() && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
                 return false;
             }
             return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty();
@@ -165,15 +174,19 @@ private boolean isDefault() {
         @Override
         public SourceFieldMapper build() {
             if (enabled.getValue().explicit()) {
-                if (indexMode == IndexMode.TIME_SERIES) {
-                    throw new MapperParsingException("Time series indices only support synthetic source");
+                if (indexMode != null && indexMode.isSyntheticSourceEnabled()) {
+                    throw new MapperParsingException("Indices with with index mode [" + indexMode + "] only support synthetic source");
                 }
                 if (mode.get() != null) {
                     throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
                 }
             }
             if (isDefault()) {
-                return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT;
+                return switch (indexMode) {
+                    case TIME_SERIES -> TSDB_DEFAULT;
+                    case LOGS -> LOGS_DEFAULT;
+                    default -> DEFAULT;
+                };
             }
             if (supportsNonDefaultParameterValues == false) {
                 List<String> disallowed = new ArrayList<>();
@@ -212,10 +225,21 @@ public SourceFieldMapper build() {
 
     }
 
-    public static final TypeParser PARSER = new ConfigurableTypeParser(
-        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES
-            ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT
-            : DEFAULT,
+    public static final TypeParser PARSER = new ConfigurableTypeParser(c -> {
+        var indexMode = c.getIndexSettings().getMode();
+        if (indexMode.isSyntheticSourceEnabled()) {
+            if (indexMode == IndexMode.TIME_SERIES) {
+                if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) {
+                    return TSDB_DEFAULT;
+                } else {
+                    return TSDB_LEGACY_DEFAULT;
+                }
+            } else if (indexMode == IndexMode.LOGS) {
+                return LOGS_DEFAULT;
+            }
+        }
+        return DEFAULT;
+    },
         c -> new Builder(
             c.getIndexSettings().getMode(),
             c.getSettings(),
@@ -323,6 +347,7 @@ public void preParse(DocumentParserContext context) throws IOException {
         final BytesReference adaptedSource = applyFilters(originalSource, contentType);
 
         if (adaptedSource != null) {
+            assert indexMode == null || indexMode.isSyntheticSourceEnabled() == false;
             final BytesRef ref = adaptedSource.toBytesRef();
             context.doc().add(new StoredField(fieldType().name(), ref.bytes, ref.offset, ref.length));
         }
diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
index fd73a8c9f8f52..caddc7d5ea5af 100644
--- a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
@@ -10,12 +10,13 @@
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.MapperServiceTestCase;
-import org.hamcrest.Matchers;
+import org.elasticsearch.test.ESTestCase;
 
-public class LogsIndexModeTests extends MapperServiceTestCase {
+import static org.hamcrest.Matchers.equalTo;
+
+public class LogsIndexModeTests extends ESTestCase {
     public void testLogsIndexModeSetting() {
-        assertThat(IndexSettings.MODE.get(buildSettings()), Matchers.equalTo(IndexMode.LOGS));
+        assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGS));
     }
 
     public void testSortField() {
@@ -24,8 +25,10 @@ public void testSortField() {
             .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
     }
 
     public void testSortMode() {
@@ -35,9 +38,11 @@ public void testSortMode() {
             .put(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "max")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("max", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("max", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey())));
     }
 
     public void testSortOrder() {
@@ -47,9 +52,11 @@ public void testSortOrder() {
             .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("desc", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("desc", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey())));
     }
 
     public void testSortMissing() {
@@ -59,9 +66,11 @@ public void testSortMissing() {
             .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("_last", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("_last", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey())));
     }
 
     private Settings buildSettings() {
diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
index 74657842488b5..525fa31673494 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat;
@@ -61,6 +62,28 @@ public class PerFieldMapperCodecTests extends ESTestCase {
         }
         """;
 
+    private static final String MAPPING_3 = """
+        {
+            "_data_stream_timestamp": {
+                "enabled": true
+            },
+            "properties": {
+                "@timestamp": {
+                    "type": "date"
+                },
+                "hostname": {
+                    "type": "keyword"
+                },
+                "response_size": {
+                    "type": "long"
+                },
+                "message": {
+                    "type": "text"
+                }
+            }
+        }
+        """;
+
     public void testUseBloomFilter() throws IOException {
         PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, randomBoolean(), false);
         assertThat(perFieldMapperCodec.useBloomFilter("_id"), is(true));
@@ -103,13 +126,13 @@ public void testDoNotUseES87TSDBEncodingForTimestampFieldNonTimeSeriesIndex() th
     }
 
     public void testEnableES87TSDBCodec() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, true, MAPPING_1);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.TIME_SERIES, MAPPING_1);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
     }
 
     public void testDisableES87TSDBCodec() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, true, MAPPING_1);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, IndexMode.TIME_SERIES, MAPPING_1);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
     }
@@ -144,31 +167,37 @@ private PerFieldFormatSupplier createFormatSupplier(boolean timestampField, bool
     }
 
     public void testUseES87TSDBEncodingSettingDisabled() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, true, MAPPING_2);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, IndexMode.TIME_SERIES, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
     }
 
     public void testUseTimeSeriesModeDisabledCodecDisabled() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, false, MAPPING_2);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.STANDARD, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
     }
 
     public void testUseTimeSeriesModeAndCodecEnabled() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, true, MAPPING_2);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.TIME_SERIES, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
     }
 
-    private PerFieldFormatSupplier createFormatSupplier(boolean enableES87TSDBCodec, boolean timeSeries, String mapping)
-        throws IOException {
+    public void testLogsIndexMode() throws IOException {
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGS, MAPPING_3);
+        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
+        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("hostname")), is(true));
+        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("response_size")), is(true));
+    }
+
+    private PerFieldFormatSupplier createFormatSupplier(boolean enableES87TSDBCodec, IndexMode mode, String mapping) throws IOException {
         Settings.Builder settings = Settings.builder();
-        if (timeSeries) {
-            settings.put(IndexSettings.MODE.getKey(), "time_series");
+        settings.put(IndexSettings.MODE.getKey(), mode);
+        if (mode == IndexMode.TIME_SERIES) {
             settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field");
         }
         settings.put(IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING.getKey(), enableES87TSDBCodec);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
index 802a18645eab6..d0350c1d92a83 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
@@ -243,6 +243,13 @@ public void testSyntheticSourceInTimeSeries() throws IOException {
         assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString());
     }
 
+    public void testSyntheticSourceWithLogsIndexMode() throws IOException {
+        XContentBuilder mapping = fieldMapping(b -> { b.field("type", "keyword"); });
+        DocumentMapper mapper = createLogsModeDocumentMapper(mapping);
+        assertTrue(mapper.sourceMapper().isSynthetic());
+        assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString());
+    }
+
     public void testSupportsNonDefaultParameterValues() throws IOException {
         Settings settings = Settings.builder().put(SourceFieldMapper.LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, false).build();
         {
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
index 7d13e33be0db7..ca49c1b8a6f7d 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
@@ -148,6 +148,11 @@ protected final DocumentMapper createTimeSeriesModeDocumentMapper(XContentBuilde
         return createMapperService(settings, mappings).documentMapper();
     }
 
+    protected final DocumentMapper createLogsModeDocumentMapper(XContentBuilder mappings) throws IOException {
+        Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), "logs").build();
+        return createMapperService(settings, mappings).documentMapper();
+    }
+
     protected final DocumentMapper createDocumentMapper(IndexVersion version, XContentBuilder mappings) throws IOException {
         return createMapperService(version, mappings).documentMapper();
     }

From 2109adc86ad2f470f29565b2556b4f65202b79c6 Mon Sep 17 00:00:00 2001
From: Pooya Salehi <pxsalehi@users.noreply.github.com>
Date: Tue, 4 Jun 2024 16:37:28 +0200
Subject: [PATCH 131/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/101598

---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index fd1999d201bf0..3421d4d2b657f 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -54,6 +54,9 @@ tests:
 - class: "org.elasticsearch.upgrades.AggregationsIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109322"
   method: "testTerms"
+- class: "org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/101598"
+  method: "testFeatureImportanceValues"
 
 # Examples:
 #

From 04124165ecbf7bcbc4dbf7fc6bafdc71e5989116 Mon Sep 17 00:00:00 2001
From: Jedr Blaszyk <jedrazb@gmail.com>
Date: Tue, 4 Jun 2024 16:38:18 +0200
Subject: [PATCH 132/208] [Docs][Connector API] Add documentation for update
 featueres endpoint (#109346)

---
 .../connector/apis/connector-apis.asciidoc    |   3 +
 .../update-connector-features-api.asciidoc    | 138 ++++++++++++++++++
 2 files changed, 141 insertions(+)
 create mode 100644 docs/reference/connector/apis/update-connector-features-api.asciidoc

diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc
index 9d23fcf8d336e..41186ff6326f2 100644
--- a/docs/reference/connector/apis/connector-apis.asciidoc
+++ b/docs/reference/connector/apis/connector-apis.asciidoc
@@ -40,6 +40,8 @@ beta:[]
 beta:[]
 * <<update-connector-configuration-api>>
 beta:[]
+* <<update-connector-features-api>>
+beta:[]
 * <<update-connector-filtering-api>>
 beta:[]
 * <<update-connector-index-name-api>>
@@ -120,6 +122,7 @@ include::list-connectors-api.asciidoc[]
 include::update-connector-api-key-id-api.asciidoc[]
 include::update-connector-configuration-api.asciidoc[]
 include::update-connector-index-name-api.asciidoc[]
+include::update-connector-features-api.asciidoc[]
 include::update-connector-filtering-api.asciidoc[]
 include::update-connector-name-description-api.asciidoc[]
 include::update-connector-pipeline-api.asciidoc[]
diff --git a/docs/reference/connector/apis/update-connector-features-api.asciidoc b/docs/reference/connector/apis/update-connector-features-api.asciidoc
new file mode 100644
index 0000000000000..0d3457b9bd584
--- /dev/null
+++ b/docs/reference/connector/apis/update-connector-features-api.asciidoc
@@ -0,0 +1,138 @@
+[[update-connector-features-api]]
+=== Update connector features API
+++++
+<titleabbrev>Update connector features</titleabbrev>
+++++
+
+beta::[]
+
+Manages the `features` of a connector. This endpoint can be used to control the following aspects of a connector:
+
+* document-level security
+* incremental syncs
+* advanced sync rules
+* basic sync rules
+
+Normally, the running connector service automatically manages these features. However, you can use this API to override the default behavior.
+
+To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^].
+
+[[update-connector-features-api-request]]
+==== {api-request-title}
+
+`PUT _connector/<connector_id>/_features`
+
+[[update-connector-features-api-prereq]]
+==== {api-prereq-title}
+
+* To sync data using self-managed connectors, you need to deploy the {enterprise-search-ref}/build-connector.html[Elastic connector service] on your own infrastructure. This service runs automatically on Elastic Cloud for native connectors.
+* The `connector_id` parameter should reference an existing connector.
+
+[[update-connector-features-api-path-params]]
+==== {api-path-parms-title}
+
+`<connector_id>`::
+(Required, string)
+
+[role="child_attributes"]
+[[update-connector-features-api-request-body]]
+==== {api-request-body-title}
+
+`features`::
+(Required, object) An object containing connector features.
+
+* `document_level_security` (Optional, object) Controls whether document-level security is enabled with the `enabled` flag.
+* `incremental_sync` (Optional, object) Controls whether incremental syncs are enabled with the `enabled` flag.
+* `native_connector_api_keys`(Optional, object) Controls whether native connector API keys are enabled with the `enabled` flag.
+* `sync_rules` (Optional, object) Controls sync rules.
+**  `advanced` (Optional, object) Controls whether advanced sync rules are enabled with the `enabled` flag.
+**  `basic`(Optional, object) Controls whether basic sync rules are enabled with the `enabled` flag.
+
+
+
+[[update-connector-features-api-response-codes]]
+==== {api-response-codes-title}
+
+`200`::
+Connector `features` was successfully updated.
+
+`400`::
+The `connector_id` was not provided or the request payload was malformed.
+
+`404` (Missing resources)::
+No connector matching `connector_id` could be found.
+
+[[update-connector-features-api-example]]
+==== {api-examples-title}
+
+The following example updates the `features` field for the connector with ID `my-connector`:
+
+////
+[source, console]
+--------------------------------------------------
+PUT _connector/my-connector
+{
+  "index_name": "search-google-drive",
+  "name": "My Connector",
+  "service_type": "google_drive"
+}
+--------------------------------------------------
+// TESTSETUP
+
+[source,console]
+--------------------------------------------------
+DELETE _connector/my-connector
+--------------------------------------------------
+// TEARDOWN
+////
+
+[source,console]
+----
+PUT _connector/my-connector/_features
+{
+  "features": {
+    "document_level_security": {
+      "enabled": true
+    },
+    "incremental_sync": {
+      "enabled": true
+    },
+    "sync_rules": {
+      "advanced": {
+        "enabled": false
+      },
+      "basic": {
+        "enabled": true
+      }
+    }
+  }
+}
+----
+
+[source,console-result]
+----
+{
+    "result": "updated"
+}
+----
+
+The endpoint supports partial updates of the `features` field. For example, to update only the `document_level_security` feature, you can send the following request:
+
+[source,console]
+----
+PUT _connector/my-connector/_features
+{
+  "features": {
+    "document_level_security": {
+      "enabled": true
+    }
+  }
+}
+----
+
+[source,console-result]
+----
+{
+    "result": "updated"
+}
+----

From be502cc319a7032e516818fc42a17d2b82830923 Mon Sep 17 00:00:00 2001
From: Pooya Salehi <pxsalehi@users.noreply.github.com>
Date: Tue, 4 Jun 2024 16:40:03 +0200
Subject: [PATCH 133/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/102717

---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index 3421d4d2b657f..628c57301f3bf 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -57,6 +57,9 @@ tests:
 - class: "org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT"
   issue: "https://github.com/elastic/elasticsearch/issues/101598"
   method: "testFeatureImportanceValues"
+- class: "org.elasticsearch.client.RestClientSingleHostIntegTests"
+  issue: "https://github.com/elastic/elasticsearch/issues/102717"
+  method: "testRequestResetAndAbort"
 
 # Examples:
 #

From 6955427699e656f6517e7e1b76a21f2cc8686c06 Mon Sep 17 00:00:00 2001
From: Luigi Dell'Aquila <luigi.dellaquila@gmail.com>
Date: Tue, 4 Jun 2024 18:37:50 +0200
Subject: [PATCH 134/208] ES|QL: remove some unused code from ESQL-CORE 
 (#109343)

Removing the unused ESQL-CORE version of PreAnalyzer and some unused
analyzer rules
---
 .../esql/core/analyzer/AnalyzerRules.java     | 108 ------------------
 .../xpack/esql/core/analyzer/PreAnalyzer.java |  51 ---------
 .../esql/core/analyzer/PreAnalyzerTests.java  |  57 ---------
 .../resources/forbidden/ql-signatures.txt     |   1 -
 4 files changed, 217 deletions(-)
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/PreAnalyzer.java
 delete mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/analyzer/PreAnalyzerTests.java

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/AnalyzerRules.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/AnalyzerRules.java
index 2bc150db64950..ce188511fe7bc 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/AnalyzerRules.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/AnalyzerRules.java
@@ -8,24 +8,10 @@
 package org.elasticsearch.xpack.esql.core.analyzer;
 
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
-import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
-import org.elasticsearch.xpack.esql.core.expression.function.Function;
-import org.elasticsearch.xpack.esql.core.expression.function.FunctionDefinition;
-import org.elasticsearch.xpack.esql.core.expression.function.FunctionRegistry;
-import org.elasticsearch.xpack.esql.core.expression.function.UnresolvedFunction;
-import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Equals;
-import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.rule.ParameterizedRule;
 import org.elasticsearch.xpack.esql.core.rule.Rule;
-import org.elasticsearch.xpack.esql.core.session.Configuration;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
-import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
-import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -34,48 +20,10 @@
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
-import static java.util.Arrays.asList;
 import static java.util.Collections.singletonList;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
 
 public final class AnalyzerRules {
 
-    public static class AddMissingEqualsToBoolField extends AnalyzerRule<Filter> {
-
-        @Override
-        protected LogicalPlan rule(Filter filter) {
-            if (filter.resolved() == false) {
-                return filter;
-            }
-            // check the condition itself
-            Expression condition = replaceRawBoolFieldWithEquals(filter.condition());
-            // otherwise look for binary logic
-            if (condition == filter.condition()) {
-                condition = condition.transformUp(
-                    BinaryLogic.class,
-                    b -> b.replaceChildren(asList(replaceRawBoolFieldWithEquals(b.left()), replaceRawBoolFieldWithEquals(b.right())))
-                );
-            }
-
-            if (condition != filter.condition()) {
-                filter = filter.with(condition);
-            }
-            return filter;
-        }
-
-        private static Expression replaceRawBoolFieldWithEquals(Expression e) {
-            if (e instanceof FieldAttribute && e.dataType() == BOOLEAN) {
-                e = new Equals(e.source(), e, Literal.of(e, Boolean.TRUE));
-            }
-            return e;
-        }
-
-        @Override
-        protected boolean skipResolved() {
-            return false;
-        }
-    }
-
     public abstract static class AnalyzerRule<SubPlan extends LogicalPlan> extends Rule<SubPlan, LogicalPlan> {
 
         // transformUp (post-order) - that is first children and then the node
@@ -123,24 +71,6 @@ protected LogicalPlan rule(LogicalPlan plan) {
         protected abstract LogicalPlan doRule(LogicalPlan plan);
     }
 
-    public static Function resolveFunction(UnresolvedFunction uf, Configuration configuration, FunctionRegistry functionRegistry) {
-        Function f = null;
-        if (uf.analyzed()) {
-            f = uf;
-        } else if (uf.childrenResolved() == false) {
-            f = uf;
-        } else {
-            String functionName = functionRegistry.resolveAlias(uf.name());
-            if (functionRegistry.functionExists(functionName) == false) {
-                f = uf.missing(functionName, functionRegistry.listFunctions());
-            } else {
-                FunctionDefinition def = functionRegistry.resolveFunction(functionName);
-                f = uf.buildResolved(configuration, def);
-            }
-        }
-        return f;
-    }
-
     public static List<Attribute> maybeResolveAgainstList(
         UnresolvedAttribute u,
         Collection<Attribute> attrList,
@@ -218,42 +148,4 @@ public static List<Attribute> maybeResolveAgainstList(
             )
         );
     }
-
-    public static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute named, boolean allowCompound) {
-        // if it's a object/compound type, keep it unresolved with a nice error message
-        if (named instanceof FieldAttribute fa) {
-
-            // incompatible mappings
-            if (fa.field() instanceof InvalidMappedField imf) {
-                named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] due to ambiguities being " + imf.errorMessage());
-            }
-            // unsupported types
-            else if (DataTypes.isUnsupported(fa.dataType())) {
-                UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
-                if (unsupportedField.hasInherited()) {
-                    named = u.withUnresolvedMessage(
-                        "Cannot use field ["
-                            + fa.name()
-                            + "] with unsupported type ["
-                            + unsupportedField.getOriginalType()
-                            + "] in hierarchy (field ["
-                            + unsupportedField.getInherited()
-                            + "])"
-                    );
-                } else {
-                    named = u.withUnresolvedMessage(
-                        "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]"
-                    );
-                }
-            }
-            // compound fields
-            else if (allowCompound == false && DataTypes.isPrimitive(fa.dataType()) == false) {
-                named = u.withUnresolvedMessage(
-                    "Cannot use field [" + fa.name() + "] type [" + fa.dataType().typeName() + "] only its subfields"
-                );
-            }
-        }
-        // make sure to copy the resolved attribute with the proper location
-        return named.withLocation(u.source());
-    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/PreAnalyzer.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/PreAnalyzer.java
deleted file mode 100644
index bf40370b5fe4f..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/PreAnalyzer.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.analyzer;
-
-import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.core.plan.logical.UnresolvedRelation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static java.util.Collections.emptyList;
-
-// Since the pre-analyzer only inspect (and does NOT transform) the tree
-// it is not built as a rule executor.
-// Further more it applies 'the rules' only once and needs to return some
-// state back.
-public class PreAnalyzer {
-
-    public static class PreAnalysis {
-        public static final PreAnalysis EMPTY = new PreAnalysis(emptyList());
-
-        public final List<TableInfo> indices;
-
-        public PreAnalysis(List<TableInfo> indices) {
-            this.indices = indices;
-        }
-    }
-
-    public PreAnalysis preAnalyze(LogicalPlan plan) {
-        if (plan.analyzed()) {
-            return PreAnalysis.EMPTY;
-        }
-
-        return doPreAnalyze(plan);
-    }
-
-    private static PreAnalysis doPreAnalyze(LogicalPlan plan) {
-        List<TableInfo> indices = new ArrayList<>();
-
-        plan.forEachUp(UnresolvedRelation.class, p -> indices.add(new TableInfo(p.table(), p.frozen())));
-
-        // mark plan as preAnalyzed (if it were marked, there would be no analysis)
-        plan.forEachUp(LogicalPlan::setPreAnalyzed);
-
-        return new PreAnalysis(indices);
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/analyzer/PreAnalyzerTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/analyzer/PreAnalyzerTests.java
deleted file mode 100644
index f48d766c0dedb..0000000000000
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/analyzer/PreAnalyzerTests.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.analyzer;
-
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.esql.core.analyzer.PreAnalyzer.PreAnalysis;
-import org.elasticsearch.xpack.esql.core.expression.Literal;
-import org.elasticsearch.xpack.esql.core.plan.TableIdentifier;
-import org.elasticsearch.xpack.esql.core.plan.logical.Limit;
-import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.core.plan.logical.UnresolvedRelation;
-
-import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
-
-public class PreAnalyzerTests extends ESTestCase {
-
-    private PreAnalyzer preAnalyzer = new PreAnalyzer();
-
-    public void testBasicIndex() {
-        LogicalPlan plan = new UnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "index"), null, false);
-        PreAnalysis result = preAnalyzer.preAnalyze(plan);
-        assertThat(plan.preAnalyzed(), is(true));
-        assertThat(result.indices, hasSize(1));
-        assertThat(result.indices.get(0).id().cluster(), nullValue());
-        assertThat(result.indices.get(0).id().index(), is("index"));
-    }
-
-    public void testBasicIndexWithCatalog() {
-        LogicalPlan plan = new UnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, "elastic", "index"), null, false);
-        PreAnalysis result = preAnalyzer.preAnalyze(plan);
-        assertThat(plan.preAnalyzed(), is(true));
-        assertThat(result.indices, hasSize(1));
-        assertThat(result.indices.get(0).id().cluster(), is("elastic"));
-        assertThat(result.indices.get(0).id().index(), is("index"));
-    }
-
-    public void testComplicatedQuery() {
-        LogicalPlan plan = new Limit(
-            EMPTY,
-            new Literal(EMPTY, 10, INTEGER),
-            new UnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, "aaa"), null, false)
-        );
-        PreAnalysis result = preAnalyzer.preAnalyze(plan);
-        assertThat(plan.preAnalyzed(), is(true));
-        assertThat(result.indices, hasSize(1));
-        assertThat(result.indices.get(0).id().cluster(), nullValue());
-        assertThat(result.indices.get(0).id().index(), is("aaa"));
-    }
-}
diff --git a/x-pack/plugin/esql/src/main/resources/forbidden/ql-signatures.txt b/x-pack/plugin/esql/src/main/resources/forbidden/ql-signatures.txt
index 95e452533dae3..5371b35f4e033 100644
--- a/x-pack/plugin/esql/src/main/resources/forbidden/ql-signatures.txt
+++ b/x-pack/plugin/esql/src/main/resources/forbidden/ql-signatures.txt
@@ -2,5 +2,4 @@ org.elasticsearch.xpack.esql.core.plan.logical.Aggregate @ use @org.elasticsearc
 org.elasticsearch.xpack.esql.core.plan.logical.EsRelation @ use @org.elasticsearch.xpack.esql.plan.logical.EsRelation instead
 org.elasticsearch.xpack.esql.core.plan.logical.Project @ use @org.elasticsearch.xpack.esql.plan.logical.Project instead
 org.elasticsearch.xpack.esql.core.plan.logical.UnresolvedRelation @ use @org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation instead
-org.elasticsearch.xpack.esql.core.analyzer.PreAnalyzer @ use @org.elasticsearch.xpack.esql.analysis.PreAnalyzer instead
 org.elasticsearch.xpack.esql.core.expression.function.Functions @ use @org.elasticsearch.xpack.esql.expression.function.Functions instead

From d9941f628589d3de9aef8e74a9369c396b8bcf9b Mon Sep 17 00:00:00 2001
From: Joe Gallo <joe.gallo@elastic.co>
Date: Tue, 4 Jun 2024 12:48:19 -0400
Subject: [PATCH 135/208] Ingest geoip new databases release highlight
 (#109355)

---
 docs/changelog/107377.yaml                    |  8 ++++++
 docs/changelog/108639.yaml                    | 23 ---------------
 docs/changelog/108651.yaml                    | 24 ----------------
 docs/changelog/108683.yaml                    | 28 +++++--------------
 .../ingest/processors/geoip.asciidoc          |  1 +
 5 files changed, 16 insertions(+), 68 deletions(-)

diff --git a/docs/changelog/107377.yaml b/docs/changelog/107377.yaml
index b9fea61d38a0a..a97f8b78dcce9 100644
--- a/docs/changelog/107377.yaml
+++ b/docs/changelog/107377.yaml
@@ -3,3 +3,11 @@ summary: Add support for the 'Enterprise' database to the geoip processor
 area: Ingest Node
 type: enhancement
 issues: []
+highlight:
+  title: "Preview: Support for the 'Anonymous IP' and 'Enterprise' databases in the geoip processor"
+  body: |-
+    As a Technical Preview, the {ref}/geoip-processor.html[`geoip`] processor can now use the commercial
+    https://www.maxmind.com/en/solutions/geoip2-enterprise-product-suite/enterprise-database[GeoIP2 'Enterprise']
+    and
+    https://www.maxmind.com/en/solutions/geoip2-enterprise-product-suite/anonymous-ip-database[GeoIP2 'Anonymous IP']
+    databases from MaxMind.
diff --git a/docs/changelog/108639.yaml b/docs/changelog/108639.yaml
index 586270c3c761c..e4964cbeb0285 100644
--- a/docs/changelog/108639.yaml
+++ b/docs/changelog/108639.yaml
@@ -3,26 +3,3 @@ summary: Add support for the 'Domain' database to the geoip processor
 area: Ingest Node
 type: enhancement
 issues: []
-highlight:
-  title: Add support for the 'Domain' database to the geoip processor
-  body: |-
-    Follow on to #107287 and #107377
-
-    Adds support for the ['GeoIP2
-    Domain'](https://dev.maxmind.com/geoip/docs/databases/domain) database
-    from MaxMind to the `geoip` processor.
-
-    The `geoip` processor will automatically download the [various
-    'GeoLite2'
-    databases](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data),
-    but the 'GeoIP2 Domain' database is not a 'GeoLite2' database -- it's a
-    commercial database available to those with a suitable license from
-    MaxMind.
-
-    The support that is being added for it in this PR is in line with the
-    support that we already have for MaxMind's 'GeoIP2 City' and 'GeoIP2
-    Country' databases -- that is, one would need to arrange their own
-    download management via some custom endpoint or otherwise arrange for
-    the relevant file(s) to be in the `$ES_CONFIG/ingest-geoip` directory on
-    the nodes of the cluster.
-  notable: true
diff --git a/docs/changelog/108651.yaml b/docs/changelog/108651.yaml
index e629c114dac51..227c464909d50 100644
--- a/docs/changelog/108651.yaml
+++ b/docs/changelog/108651.yaml
@@ -3,27 +3,3 @@ summary: Add support for the 'ISP' database to the geoip processor
 area: Ingest Node
 type: enhancement
 issues: []
-highlight:
-  title: Add support for the 'ISP' database to the geoip processor
-  body: |-
-    Follow on to https://github.com/elastic/elasticsearch/pull/107287,
-    https://github.com/elastic/elasticsearch/pull/107377, and
-    https://github.com/elastic/elasticsearch/pull/108639
-
-    Adds support for the ['GeoIP2
-    ISP'](https://dev.maxmind.com/geoip/docs/databases/isp) database from
-    MaxMind to the geoip processor.
-
-    The geoip processor will automatically download the [various 'GeoLite2'
-    databases](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data),
-    but the 'GeoIP2 ISP' database is not a 'GeoLite2' database -- it's a
-    commercial database available to those with a suitable license from
-    MaxMind.
-
-    The support that is being added for it in this PR is in line with the
-    support that we already have for MaxMind's 'GeoIP2 City' and 'GeoIP2
-    Country' databases -- that is, one would need to arrange their own
-    download management via some custom endpoint or otherwise arrange for
-    the relevant file(s) to be in the $ES_CONFIG/ingest-geoip directory on
-    the nodes of the cluster.
-  notable: true
diff --git a/docs/changelog/108683.yaml b/docs/changelog/108683.yaml
index ad796fb9b25c7..b9e7df5fefc18 100644
--- a/docs/changelog/108683.yaml
+++ b/docs/changelog/108683.yaml
@@ -4,25 +4,11 @@ area: Ingest Node
 type: enhancement
 issues: []
 highlight:
-  title: Add support for the 'Connection Type' database to the geoip processor
+  title: "Preview: Support for the 'Connection Type, 'Domain', and 'ISP' databases in the geoip processor"
   body: |-
-    Follow on to #107287, #107377, #108639, and #108651
-
-    Adds support for the ['GeoIP2 Connection
-    Type'](https://dev.maxmind.com/geoip/docs/databases/connection-type)
-    database from MaxMind to the `geoip` processor.
-
-    The `geoip` processor will automatically download the [various
-    'GeoLite2'
-    databases](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data),
-    but the 'GeoIP2 Connection Type' database is not a 'GeoLite2' database
-    -- it's a commercial database available to those with a suitable license
-    from MaxMind.
-
-    The support that is being added for it in this PR is in line with the
-    support that we already have for MaxMind's 'GeoIP2 City' and 'GeoIP2
-    Country' databases -- that is, one would need to arrange their own
-    download management via some custom endpoint or otherwise arrange for
-    the relevant file(s) to be in the `$ES_CONFIG/ingest-geoip` directory on
-    the nodes of the cluster.
-  notable: true
+    As a Technical Preview, the {ref}/geoip-processor.html[`geoip`] processor can now use the commercial
+    https://dev.maxmind.com/geoip/docs/databases/connection-type[GeoIP2 'Connection Type'],
+    https://dev.maxmind.com/geoip/docs/databases/domain[GeoIP2 'Domain'],
+    and
+    https://dev.maxmind.com/geoip/docs/databases/isp[GeoIP2 'ISP']
+    databases from MaxMind.
diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc
index e85165a28dcb4..738ac234d6162 100644
--- a/docs/reference/ingest/processors/geoip.asciidoc
+++ b/docs/reference/ingest/processors/geoip.asciidoc
@@ -72,6 +72,7 @@ depend on what has been found and which properties were configured in `propertie
 `residential_proxy`, `domain`, `isp`, `isp_organization`, `mobile_country_code`, `mobile_network_code`, `user_type`, and
 `connection_type`. The fields actually added  depend on what has been found and which properties were configured in `properties`.
 
+preview::["Do not use the GeoIP2 Anonymous IP, GeoIP2 Connection Type, GeoIP2 Domain, GeoIP2 ISP, and GeoIP2 Enterprise databases in production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
 
 Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:
 

From 4d91f0f3c0f870b9d5dd0e325b30a03064b77777 Mon Sep 17 00:00:00 2001
From: Jake Landis <jake.landis@elastic.co>
Date: Tue, 4 Jun 2024 12:19:23 -0500
Subject: [PATCH 136/208] Fix test->testAutomatonPatterns dependency (#109323)

As-is, the testAutomatonPatterns test(s) always runs before any other
test task. This made sense thinking about the entire test suite, but is
annoying when trying to run an isolated single unit test, where both
tests end up running.

This commit runs the testAutomatonPatterns with the `check` task instead
of the `test` task.

related: https://github.com/elastic/elasticsearch/pull/108590
---
 x-pack/plugin/core/build.gradle | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index f0a00b7aa7e75..116f864695e15 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -191,5 +191,6 @@ tasks.register('testAutomatonPatterns', Test) {
 
 tasks.named('test').configure {
   exclude '**/AutomatonPatternsTests.class'
-  dependsOn testAutomatonPatterns //to ensure testAutomatonPatterns are run with the test task
 }
+
+tasks.named("check").configure { dependsOn "testAutomatonPatterns" }

From 21952c7e36a83cbd94da42390ba44d863bc5d165 Mon Sep 17 00:00:00 2001
From: Luigi Dell'Aquila <luigi.dellaquila@gmail.com>
Date: Tue, 4 Jun 2024 19:33:14 +0200
Subject: [PATCH 137/208] ES|QL: add geo tests for mv_dedupe (#109342)

Adding more unit tests for MV_DEDUPE function, covering geo_point,
geo_shape, cartesian_point and cartesian_shape. This also adds docs for
Kibana.

Fixes https://github.com/elastic/elasticsearch/issues/108982
---
 .../kibana/definition/mv_dedupe.json          | 48 +++++++++++++++++++
 .../esql/functions/types/mv_dedupe.asciidoc   |  4 ++
 .../src/main/resources/meta.csv-spec          |  6 +--
 .../function/scalar/multivalue/MvDedupe.java  | 32 +++++++++++--
 .../scalar/multivalue/MvDedupeTests.java      |  6 +++
 5 files changed, 90 insertions(+), 6 deletions(-)

diff --git a/docs/reference/esql/functions/kibana/definition/mv_dedupe.json b/docs/reference/esql/functions/kibana/definition/mv_dedupe.json
index c8d221f933de1..7ab287bc94d34 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_dedupe.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_dedupe.json
@@ -17,6 +17,30 @@
       "variadic" : false,
       "returnType" : "boolean"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "cartesian_point",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "cartesian_point"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "cartesian_shape",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "cartesian_shape"
+    },
     {
       "params" : [
         {
@@ -41,6 +65,30 @@
       "variadic" : false,
       "returnType" : "double"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "geo_point",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "geo_point"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "geo_shape",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "geo_shape"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/types/mv_dedupe.asciidoc b/docs/reference/esql/functions/types/mv_dedupe.asciidoc
index 705745d76dbab..a6b78f781f17a 100644
--- a/docs/reference/esql/functions/types/mv_dedupe.asciidoc
+++ b/docs/reference/esql/functions/types/mv_dedupe.asciidoc
@@ -6,8 +6,12 @@
 |===
 field | result
 boolean | boolean
+cartesian_point | cartesian_point
+cartesian_shape | cartesian_shape
 datetime | datetime
 double | double
+geo_point | geo_point
+geo_shape | geo_shape
 integer | integer
 ip | ip
 keyword | keyword
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
index eff4cb05bd8c0..d2b62c479c696 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
@@ -45,7 +45,7 @@ double e()
 "double mv_avg(number:double|integer|long|unsigned_long)"
 "keyword mv_concat(string:text|keyword, delim:text|keyword)"
 "integer mv_count(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
-"boolean|date|double|integer|ip|keyword|long|text|version mv_dedupe(field:boolean|date|double|integer|ip|keyword|long|text|version)"
+"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_dedupe(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version)"
 "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version mv_first(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
 "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version mv_last(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
 "boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version mv_max(field:boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version)"
@@ -160,7 +160,7 @@ min           |number                              |"double|integer|long"
 mv_avg        |number                              |"double|integer|long|unsigned_long"                                                                                               |Multivalue expression.
 mv_concat     |[string, delim]                     |["text|keyword", "text|keyword"]                                                                                                  |[Multivalue expression., Delimiter.]
 mv_count      |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Multivalue expression.
-mv_dedupe     |field                               |"boolean|date|double|integer|ip|keyword|long|text|version"                                                                        |Multivalue expression.
+mv_dedupe     |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"                    |Multivalue expression.
 mv_first      |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Multivalue expression.
 mv_last       |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Multivalue expression.
 mv_max        |field                               |"boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version"                                                          |Multivalue expression.
@@ -393,7 +393,7 @@ min           |"double|integer|long"
 mv_avg        |double                                                                                                                      |false                       |false           |false
 mv_concat     |keyword                                                                                                                     |[false, false]              |false           |false
 mv_count      |integer                                                                                                                     |false                       |false           |false
-mv_dedupe     |"boolean|date|double|integer|ip|keyword|long|text|version"                                                                  |false                       |false           |false
+mv_dedupe     |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"              |false                       |false           |false
 mv_first      |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"|false                       |false           |false
 mv_last       |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"|false                       |false           |false
 mv_max        |"boolean|date|double|integer|ip|keyword|long|text|unsigned_long|version"                                                    |false                       |false           |false
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java
index 543e37f43f671..71cf759b3dbe5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java
@@ -26,9 +26,22 @@
  * Removes duplicate values from a multivalued field.
  */
 public class MvDedupe extends AbstractMultivalueFunction {
-    // @TODO: add cartesian_point, geo_point, unsigned_long
+    // @TODO: add unsigned_long
     @FunctionInfo(
-        returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+        returnType = {
+            "boolean",
+            "cartesian_point",
+            "cartesian_shape",
+            "date",
+            "double",
+            "geo_point",
+            "geo_shape",
+            "integer",
+            "ip",
+            "keyword",
+            "long",
+            "text",
+            "version" },
         description = "Remove duplicate values from a multivalued field.",
         note = "`MV_DEDUPE` may, but won't always, sort the values in the column.",
         examples = @Example(file = "string", tag = "mv_dedupe")
@@ -37,7 +50,20 @@ public MvDedupe(
         Source source,
         @Param(
             name = "field",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
+            type = {
+                "boolean",
+                "cartesian_point",
+                "cartesian_shape",
+                "date",
+                "double",
+                "geo_point",
+                "geo_shape",
+                "integer",
+                "ip",
+                "keyword",
+                "long",
+                "text",
+                "version" },
             description = "Multivalue expression."
         ) Expression field
     ) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
index 0aed84d57004a..5f275e4c52d43 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
@@ -42,6 +43,11 @@ public static Iterable<Object[]> parameters() {
         doubles(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values.mapToObj(Double::valueOf)));
         ints(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values.mapToObj(Integer::valueOf)));
         longs(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values.mapToObj(Long::valueOf)));
+        cartesianPoints(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values));
+        cartesianShape(cases, "mv_dedupe", "MvDedupe", DataTypes.CARTESIAN_SHAPE, (size, values) -> getMatcher(values));
+        geoPoints(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values));
+        geoShape(cases, "mv_dedupe", "MvDedupe", DataTypes.GEO_SHAPE, (size, values) -> getMatcher(values));
+
         // TODO switch extraction to BigInteger so this just works.
         // unsignedLongs(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values));
         return parameterSuppliersFromTypedData(cases);

From 5f6e8f687b16c75b4fee705c8a5cd0ad8163f2e5 Mon Sep 17 00:00:00 2001
From: Luigi Dell'Aquila <luigi.dellaquila@gmail.com>
Date: Tue, 4 Jun 2024 19:42:29 +0200
Subject: [PATCH 138/208] ES|QL: add MV_APPEND function (#107001)

Adding `MV_APPEND(value1, value2)` function, that appends two values
creating a single multi-value. If one or both the inputs are
multi-values, the result is the concatenation of all the values, eg.

```
MV_APPEND([a, b], [c, d]) -> [a, b, c, d]
```

~I think for this specific case it makes sense to consider `null` values
as empty arrays, so that~ ~MV_APPEND(value, null) -> value~ ~It is
pretty uncommon for ESQL (all the other functions, apart from
`COALESCE`, short-circuit to `null` when one of the values is null), so
let's discuss this behavior.~

[EDIT] considering the feedback from Andrei, I changed this logic and
made it consistent with the other functions: now if one of the
parameters is null, the function returns null
---
 .../functions/description/mv_append.asciidoc  |   5 +
 .../kibana/definition/mv_append.json          | 242 ++++++++++++++
 .../esql/functions/kibana/docs/mv_append.md   |   7 +
 .../esql/functions/layout/mv_append.asciidoc  |  14 +
 .../functions/parameters/mv_append.asciidoc   |   9 +
 .../esql/functions/signature/mv_append.svg    |   1 +
 .../esql/functions/types/mv_append.asciidoc   |  21 ++
 .../mapping-multi-field-variation.json        |   2 +
 .../src/main/resources/date.csv-spec          |  17 +
 .../src/main/resources/floats.csv-spec        |  19 ++
 .../src/main/resources/meta.csv-spec          |   6 +-
 .../src/main/resources/string.csv-spec        |  73 +++++
 .../src/main/resources/version.csv-spec       |  15 +
 .../multivalue/MvAppendBooleanEvaluator.java  | 102 ++++++
 .../multivalue/MvAppendBytesRefEvaluator.java | 103 ++++++
 .../multivalue/MvAppendDoubleEvaluator.java   | 102 ++++++
 .../multivalue/MvAppendIntEvaluator.java      | 102 ++++++
 .../multivalue/MvAppendLongEvaluator.java     | 102 ++++++
 .../xpack/esql/action/EsqlCapabilities.java   |   6 +
 .../function/EsqlFunctionRegistry.java        |   2 +
 .../function/scalar/multivalue/MvAppend.java  | 285 +++++++++++++++++
 .../xpack/esql/io/stream/PlanNamedTypes.java  |  14 +
 .../xpack/esql/analysis/AnalyzerTests.java    |  41 +++
 .../scalar/multivalue/MvAppendTests.java      | 296 ++++++++++++++++++
 24 files changed, 1585 insertions(+), 1 deletion(-)
 create mode 100644 docs/reference/esql/functions/description/mv_append.asciidoc
 create mode 100644 docs/reference/esql/functions/kibana/definition/mv_append.json
 create mode 100644 docs/reference/esql/functions/kibana/docs/mv_append.md
 create mode 100644 docs/reference/esql/functions/layout/mv_append.asciidoc
 create mode 100644 docs/reference/esql/functions/parameters/mv_append.asciidoc
 create mode 100644 docs/reference/esql/functions/signature/mv_append.svg
 create mode 100644 docs/reference/esql/functions/types/mv_append.asciidoc
 create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java
 create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java
 create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java
 create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java
 create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java

diff --git a/docs/reference/esql/functions/description/mv_append.asciidoc b/docs/reference/esql/functions/description/mv_append.asciidoc
new file mode 100644
index 0000000000000..26b549713e301
--- /dev/null
+++ b/docs/reference/esql/functions/description/mv_append.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Concatenates values of two multi-value fields.
diff --git a/docs/reference/esql/functions/kibana/definition/mv_append.json b/docs/reference/esql/functions/kibana/definition/mv_append.json
new file mode 100644
index 0000000000000..8ee4e7297cc3a
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/mv_append.json
@@ -0,0 +1,242 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "mv_append",
+  "description" : "Concatenates values of two multi-value fields.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "cartesian_point",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "cartesian_point",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "cartesian_point"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "cartesian_shape",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "cartesian_shape",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "cartesian_shape"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "datetime",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "datetime",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "datetime"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "double",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "double",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "geo_point",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "geo_point",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "geo_point"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "geo_shape",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "geo_shape",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "geo_shape"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "integer",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "integer",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "ip",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "ip",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "ip"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "long",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "long",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "text",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "text",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "text"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field1",
+          "type" : "version",
+          "optional" : false,
+          "description" : ""
+        },
+        {
+          "name" : "field2",
+          "type" : "version",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "version"
+    }
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/docs/mv_append.md b/docs/reference/esql/functions/kibana/docs/mv_append.md
new file mode 100644
index 0000000000000..36b285be1877c
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/mv_append.md
@@ -0,0 +1,7 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### MV_APPEND
+Concatenates values of two multi-value fields.
+
diff --git a/docs/reference/esql/functions/layout/mv_append.asciidoc b/docs/reference/esql/functions/layout/mv_append.asciidoc
new file mode 100644
index 0000000000000..4d4dbd7a24f9d
--- /dev/null
+++ b/docs/reference/esql/functions/layout/mv_append.asciidoc
@@ -0,0 +1,14 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-mv_append]]
+=== `MV_APPEND`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/mv_append.svg[Embedded,opts=inline]
+
+include::../parameters/mv_append.asciidoc[]
+include::../description/mv_append.asciidoc[]
+include::../types/mv_append.asciidoc[]
diff --git a/docs/reference/esql/functions/parameters/mv_append.asciidoc b/docs/reference/esql/functions/parameters/mv_append.asciidoc
new file mode 100644
index 0000000000000..e08d697c25098
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/mv_append.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`field1`::
+
+
+`field2`::
+
diff --git a/docs/reference/esql/functions/signature/mv_append.svg b/docs/reference/esql/functions/signature/mv_append.svg
new file mode 100644
index 0000000000000..0f45435425c65
--- /dev/null
+++ b/docs/reference/esql/functions/signature/mv_append.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="468" height="46" viewbox="0 0 468 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m128 0h10m32 0h10m92 0h10m32 0h10m92 0h10m32 0h5"/><rect class="s" x="5" y="5" width="128" height="36"/><text class="k" x="15" y="31">MV_APPEND</text><rect class="s" x="143" y="5" width="32" height="36" rx="7"/><text class="syn" x="153" y="31">(</text><rect class="s" x="185" y="5" width="92" height="36" rx="7"/><text class="k" x="195" y="31">field1</text><rect class="s" x="287" y="5" width="32" height="36" rx="7"/><text class="syn" x="297" y="31">,</text><rect class="s" x="329" y="5" width="92" height="36" rx="7"/><text class="k" x="339" y="31">field2</text><rect class="s" x="431" y="5" width="32" height="36" rx="7"/><text class="syn" x="441" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/types/mv_append.asciidoc b/docs/reference/esql/functions/types/mv_append.asciidoc
new file mode 100644
index 0000000000000..49dcef6dc8860
--- /dev/null
+++ b/docs/reference/esql/functions/types/mv_append.asciidoc
@@ -0,0 +1,21 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+field1 | field2 | result
+boolean | boolean | boolean
+cartesian_point | cartesian_point | cartesian_point
+cartesian_shape | cartesian_shape | cartesian_shape
+datetime | datetime | datetime
+double | double | double
+geo_point | geo_point | geo_point
+geo_shape | geo_shape | geo_shape
+integer | integer | integer
+ip | ip | ip
+keyword | keyword | keyword
+long | long | long
+text | text | text
+version | version | version
+|===
diff --git a/x-pack/plugin/esql-core/src/test/resources/mapping-multi-field-variation.json b/x-pack/plugin/esql-core/src/test/resources/mapping-multi-field-variation.json
index b5b3d42816502..5369e50dd6bb9 100644
--- a/x-pack/plugin/esql-core/src/test/resources/mapping-multi-field-variation.json
+++ b/x-pack/plugin/esql-core/src/test/resources/mapping-multi-field-variation.json
@@ -8,6 +8,8 @@
         "keyword" : { "type" : "keyword" },
         "date" :  { "type" : "date" },
         "date_nanos": { "type" : "date_nanos" },
+        "long" :  { "type" : "long" },
+        "ip" :  { "type" : "ip" },
         "unsupported" : { "type" : "ip_range" },
         "some" : {
             "properties" : {
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
index 0c808afc9d12b..63421aec35665 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
@@ -1076,6 +1076,23 @@ required_capability: agg_values
 [1955-01-21T00:00:00Z, 1957-05-23T00:00:00Z, 1959-12-03T00:00:00Z] | null
 ;
 
+
+mvAppendDates
+required_capability: fn_mv_append
+
+FROM employees
+| WHERE emp_no == 10039 OR emp_no == 10040
+| SORT emp_no
+| EVAL dates = mv_append(birth_date, hire_date)
+| KEEP emp_no, birth_date, hire_date, dates
+;
+
+emp_no:integer | birth_date:date      | hire_date:date       | dates:date
+10039          | 1959-10-01T00:00:00Z | 1988-01-19T00:00:00Z | [1959-10-01T00:00:00Z, 1988-01-19T00:00:00Z]
+10040          | null                 | 1993-02-14T00:00:00Z | null 
+;
+
+
 implicitCastingNotEqual
 required_capability: string_literal_auto_casting
 from employees | where birth_date != "1957-05-23T00:00:00Z" | keep emp_no, birth_date | sort emp_no | limit 3;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec
index bbe0df9a8cda9..e88daa63ec557 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec
@@ -523,6 +523,25 @@ required_capability: agg_values
 [1.7, 1.83, 2.05] | null
 ;
 
+
+mvAppend
+required_capability: fn_mv_append
+
+FROM employees
+| WHERE emp_no == 10008 OR emp_no == 10021
+| EVAL d = mv_append(salary_change, salary_change),
+       i = mv_append(salary_change.int, salary_change.int),
+       i2 = mv_append(emp_no, salary_change.int),
+       i3 = mv_append(emp_no, emp_no),  
+       s = mv_append(salary_change.keyword, salary_change.keyword)
+| KEEP emp_no, salary_change, d, i, i2, i3, s;
+
+emp_no:integer | salary_change:double    | d:double                                       | i:integer              | i2:integer         | i3:integer     | s:keyword
+10008          | [-2.92,0.75,3.54,12.68] | [-2.92,0.75,3.54,12.68,-2.92,0.75,3.54,12.68]  | [-2,0,3,12,-2,0,3,12]  | [10008,-2,0,3,12]  | [10008, 10008] | [-2.92,0.75,12.68,3.54,-2.92,0.75,12.68,3.54]
+10021          | null                    | null                                           | null                   | null               | [10021, 10021] | null
+;
+
+
 signumOfPositiveDouble#[skip:-8.13.99,reason:new scalar function added in 8.14]
 row d = to_double(100) | eval s = signum(d);
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
index d2b62c479c696..621944ce26e0b 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
@@ -42,6 +42,7 @@ double e()
 "double|integer|long median(number:double|integer|long)"
 "double|integer|long median_absolute_deviation(number:double|integer|long)"
 "double|integer|long min(number:double|integer|long)"
+"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_append(field1:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version, field2:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version)"
 "double mv_avg(number:double|integer|long|unsigned_long)"
 "keyword mv_concat(string:text|keyword, delim:text|keyword)"
 "integer mv_count(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
@@ -157,6 +158,7 @@ max           |number                              |"double|integer|long"
 median        |number                              |"double|integer|long"                                                                                                             |[""]
 median_absolut|number                              |"double|integer|long"                                                                                                             |[""]
 min           |number                              |"double|integer|long"                                                                                                             |[""]
+mv_append     |[field1, field2]                    |["boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version", "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"]                 | ["", ""]
 mv_avg        |number                              |"double|integer|long|unsigned_long"                                                                                               |Multivalue expression.
 mv_concat     |[string, delim]                     |["text|keyword", "text|keyword"]                                                                                                  |[Multivalue expression., Delimiter.]
 mv_count      |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Multivalue expression.
@@ -273,6 +275,7 @@ max           |The maximum value of a numeric field.
 median        |The value that is greater than half of all values and less than half of all values.
 median_absolut|The median absolute deviation, a measure of variability.
 min           |The minimum value of a numeric field.
+mv_append     |Concatenates values of two multi-value fields.
 mv_avg        |Converts a multivalued field into a single valued field containing the average of all of the values.
 mv_concat     |Converts a multivalued string expression into a single valued column containing the concatenation of all values separated by a delimiter.
 mv_count      |Converts a multivalued expression into a single valued column containing a count of the number of values.
@@ -390,6 +393,7 @@ max           |"double|integer|long"
 median        |"double|integer|long"                                                                                                       |false                       |false           |true
 median_absolut|"double|integer|long"                                                                                                       |false                       |false           |true
 min           |"double|integer|long"                                                                                                       |false                       |false           |true
+mv_append     |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"              |[false, false]              |false 		     |false
 mv_avg        |double                                                                                                                      |false                       |false           |false
 mv_concat     |keyword                                                                                                                     |[false, false]              |false           |false
 mv_count      |integer                                                                                                                     |false                       |false           |false
@@ -475,5 +479,5 @@ countFunctions#[skip:-8.14.99, reason:BIN added]
 meta functions |  stats  a = count(*), b = count(*), c = count(*) |  mv_expand c;
 
 a:long | b:long | c:long
-107    | 107    | 107
+108    | 108    | 108
 ;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
index 13616e5146949..3cb7c6ef0f594 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
@@ -1335,6 +1335,79 @@ l1:integer | l2:integer
 null       | 0
 ;
 
+
+mvAppend
+required_capability: fn_mv_append
+
+ROW a = "a", b = ["b", "c"], n = null
+| EVAL aa = mv_append(a, a), bb = mv_append(b, b), ab = mv_append(a, b), abb = mv_append(mv_append(a, b), b), na = mv_append(n, a), an = mv_append(a, n)
+;
+
+a:keyword | b:keyword | n:null | aa:keyword | bb:keyword   | ab:keyword | abb:keyword    | na:keyword | an:keyword
+a         | [b, c]    | null   |[a, a]     | [b, c, b, c] | [a, b, c]  | [a, b, c, b, c] | null       | null
+;
+
+
+mvAppendNull
+required_capability: fn_mv_append
+
+ROW a = "a", b = ["b", "c"], c = to_string(null)
+| EVAL a_null = mv_append(a, c),
+       null_a = mv_append(c, a),
+       b_null = mv_append(b, c),
+       null_b = mv_append(c, b),
+       null_null = mv_append(c, c)
+;
+
+a:keyword | b:keyword | c:keyword | a_null:keyword | null_a:keyword | b_null:keyword | null_b:keyword | null_null:keyword  
+a         | [b, c]    | null      | null           | null           | null           | null           | null
+;
+
+
+mvAppendStrings
+required_capability: fn_mv_append
+
+FROM employees
+| WHERE emp_no == 10004
+| EVAL names = mv_sort(mv_append(first_name, last_name)), 
+       two_jobs = mv_sort(mv_append(job_positions, job_positions)),
+       three_jobs = mv_sort(mv_append(job_positions, mv_append(job_positions, job_positions)))
+| KEEP emp_no, names, two_jobs, three_jobs
+;
+
+emp_no:integer | names:keyword    | two_jobs:keyword                                       | three_jobs:keyword
+10004          | ["Chirstian", "Koblick"] | ["Head Human Resources","Head Human Resources","Reporting Analyst","Reporting Analyst","Support Engineer","Support Engineer","Tech Lead","Tech Lead"]  | ["Head Human Resources","Head Human Resources","Head Human Resources","Reporting Analyst","Reporting Analyst","Reporting Analyst","Support Engineer","Support Engineer","Support Engineer","Tech Lead","Tech Lead","Tech Lead"] 
+;
+
+
+
+mvAppendStringsWhere
+required_capability: fn_mv_append
+
+FROM employees
+| EVAL two_jobs = mv_append(mv_sort(job_positions), mv_sort(job_positions)) 
+| WHERE emp_no == 10004 AND mv_slice(mv_append(mv_sort(job_positions), mv_sort(job_positions)), 6, 6) == "Support Engineer"
+| KEEP emp_no, two_jobs
+;
+
+emp_no:integer | two_jobs:keyword                       
+10004          | ["Head Human Resources","Reporting Analyst","Support Engineer","Tech Lead","Head Human Resources","Reporting Analyst","Support Engineer","Tech Lead"]   
+;
+
+mvAppendNullFields
+required_capability: fn_mv_append
+
+FROM employees
+| WHERE emp_no == 10005
+| EVAL x = mv_append(first_name, job_positions), y = mv_append(job_positions, first_name), z = mv_append(job_positions, job_positions)
+| keep emp_no, first_name, job_positions, x, y, z
+;
+
+emp_no:integer | first_name:keyword | job_positions:keyword | x:keyword | y:keyword | z:keyword
+10005          | Kyoichi            | null                  | null      | null      | null
+;         
+
+
 base64Encode#[skip:-8.13.99,reason:new base64 function added in 8.14]
 required_capability: base64_decode_encode
 
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec
index 3b6c41f883018..eb0d6d75a7d07 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec
@@ -371,6 +371,21 @@ version:version | name:keyword
           5.2.9 | mmmmm
 ;
 
+
+mvAppend
+required_capability: fn_mv_append
+
+ROW a = to_version("1.2.0"), x1 = to_version("0.0.1"), x2 = to_version("1.0.0")
+| EVAL b = mv_append(x1, x2)
+| EVAL aa = mv_append(a, a), bb = mv_append(b, b), ab = mv_append(a, b), abb = mv_append(mv_append(a, b), b)
+| KEEP a, b, aa, bb, ab, abb
+;
+
+a:version | b:version       | aa:version         | bb:version                   | ab:version             | abb:version
+1.2.0     | [0.0.1, 1.0.0]  | [1.2.0, 1.2.0]     | [0.0.1, 1.0.0, 0.0.1, 1.0.0] | [1.2.0, 0.0.1, 1.0.0]  | [1.2.0, 0.0.1, 1.0.0, 0.0.1, 1.0.0]
+;
+
+
 implictCastingEqual
 required_capability: string_literal_auto_casting_extended
 from apps | where version == "1.2.3.4" | sort name | keep name, version;
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java
new file mode 100644
index 0000000000000..c59d915fa2fe4
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java
@@ -0,0 +1,102 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvAppend}.
+ * This class is generated. Do not edit it.
+ */
+public final class MvAppendBooleanEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator field1;
+
+  private final EvalOperator.ExpressionEvaluator field2;
+
+  private final DriverContext driverContext;
+
+  public MvAppendBooleanEvaluator(Source source, EvalOperator.ExpressionEvaluator field1,
+      EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) {
+    this.warnings = new Warnings(source);
+    this.field1 = field1;
+    this.field2 = field2;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (BooleanBlock field1Block = (BooleanBlock) field1.eval(page)) {
+      try (BooleanBlock field2Block = (BooleanBlock) field2.eval(page)) {
+        return eval(page.getPositionCount(), field1Block, field2Block);
+      }
+    }
+  }
+
+  public BooleanBlock eval(int positionCount, BooleanBlock field1Block, BooleanBlock field2Block) {
+    try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        boolean allBlocksAreNulls = true;
+        if (!field1Block.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (!field2Block.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (allBlocksAreNulls) {
+          result.appendNull();
+          continue position;
+        }
+        MvAppend.process(result, p, field1Block, field2Block);
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "MvAppendBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(field1, field2);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field1;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field2;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1,
+        EvalOperator.ExpressionEvaluator.Factory field2) {
+      this.source = source;
+      this.field1 = field1;
+      this.field2 = field2;
+    }
+
+    @Override
+    public MvAppendBooleanEvaluator get(DriverContext context) {
+      return new MvAppendBooleanEvaluator(source, field1.get(context), field2.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "MvAppendBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java
new file mode 100644
index 0000000000000..c650a803a6dd4
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java
@@ -0,0 +1,103 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvAppend}.
+ * This class is generated. Do not edit it.
+ */
+public final class MvAppendBytesRefEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator field1;
+
+  private final EvalOperator.ExpressionEvaluator field2;
+
+  private final DriverContext driverContext;
+
+  public MvAppendBytesRefEvaluator(Source source, EvalOperator.ExpressionEvaluator field1,
+      EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) {
+    this.warnings = new Warnings(source);
+    this.field1 = field1;
+    this.field2 = field2;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (BytesRefBlock field1Block = (BytesRefBlock) field1.eval(page)) {
+      try (BytesRefBlock field2Block = (BytesRefBlock) field2.eval(page)) {
+        return eval(page.getPositionCount(), field1Block, field2Block);
+      }
+    }
+  }
+
+  public BytesRefBlock eval(int positionCount, BytesRefBlock field1Block,
+      BytesRefBlock field2Block) {
+    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        boolean allBlocksAreNulls = true;
+        if (!field1Block.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (!field2Block.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (allBlocksAreNulls) {
+          result.appendNull();
+          continue position;
+        }
+        MvAppend.process(result, p, field1Block, field2Block);
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "MvAppendBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(field1, field2);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field1;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field2;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1,
+        EvalOperator.ExpressionEvaluator.Factory field2) {
+      this.source = source;
+      this.field1 = field1;
+      this.field2 = field2;
+    }
+
+    @Override
+    public MvAppendBytesRefEvaluator get(DriverContext context) {
+      return new MvAppendBytesRefEvaluator(source, field1.get(context), field2.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "MvAppendBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java
new file mode 100644
index 0000000000000..07108c0522b8d
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java
@@ -0,0 +1,102 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvAppend}.
+ * This class is generated. Do not edit it.
+ */
+public final class MvAppendDoubleEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator field1;
+
+  private final EvalOperator.ExpressionEvaluator field2;
+
+  private final DriverContext driverContext;
+
+  public MvAppendDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator field1,
+      EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) {
+    this.warnings = new Warnings(source);
+    this.field1 = field1;
+    this.field2 = field2;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (DoubleBlock field1Block = (DoubleBlock) field1.eval(page)) {
+      try (DoubleBlock field2Block = (DoubleBlock) field2.eval(page)) {
+        return eval(page.getPositionCount(), field1Block, field2Block);
+      }
+    }
+  }
+
+  public DoubleBlock eval(int positionCount, DoubleBlock field1Block, DoubleBlock field2Block) {
+    try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        boolean allBlocksAreNulls = true;
+        if (!field1Block.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (!field2Block.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (allBlocksAreNulls) {
+          result.appendNull();
+          continue position;
+        }
+        MvAppend.process(result, p, field1Block, field2Block);
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "MvAppendDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(field1, field2);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field1;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field2;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1,
+        EvalOperator.ExpressionEvaluator.Factory field2) {
+      this.source = source;
+      this.field1 = field1;
+      this.field2 = field2;
+    }
+
+    @Override
+    public MvAppendDoubleEvaluator get(DriverContext context) {
+      return new MvAppendDoubleEvaluator(source, field1.get(context), field2.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "MvAppendDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java
new file mode 100644
index 0000000000000..99c5c9798a0aa
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java
@@ -0,0 +1,102 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvAppend}.
+ * This class is generated. Do not edit it.
+ */
+public final class MvAppendIntEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator field1;
+
+  private final EvalOperator.ExpressionEvaluator field2;
+
+  private final DriverContext driverContext;
+
+  public MvAppendIntEvaluator(Source source, EvalOperator.ExpressionEvaluator field1,
+      EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) {
+    this.warnings = new Warnings(source);
+    this.field1 = field1;
+    this.field2 = field2;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (IntBlock field1Block = (IntBlock) field1.eval(page)) {
+      try (IntBlock field2Block = (IntBlock) field2.eval(page)) {
+        return eval(page.getPositionCount(), field1Block, field2Block);
+      }
+    }
+  }
+
+  public IntBlock eval(int positionCount, IntBlock field1Block, IntBlock field2Block) {
+    try(IntBlock.Builder result = driverContext.blockFactory().newIntBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        boolean allBlocksAreNulls = true;
+        if (!field1Block.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (!field2Block.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (allBlocksAreNulls) {
+          result.appendNull();
+          continue position;
+        }
+        MvAppend.process(result, p, field1Block, field2Block);
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "MvAppendIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(field1, field2);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field1;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field2;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1,
+        EvalOperator.ExpressionEvaluator.Factory field2) {
+      this.source = source;
+      this.field1 = field1;
+      this.field2 = field2;
+    }
+
+    @Override
+    public MvAppendIntEvaluator get(DriverContext context) {
+      return new MvAppendIntEvaluator(source, field1.get(context), field2.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "MvAppendIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java
new file mode 100644
index 0000000000000..340a9747b14aa
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java
@@ -0,0 +1,102 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import java.lang.Override;
+import java.lang.String;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvAppend}.
+ * This class is generated. Do not edit it.
+ */
+public final class MvAppendLongEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final EvalOperator.ExpressionEvaluator field1;
+
+  private final EvalOperator.ExpressionEvaluator field2;
+
+  private final DriverContext driverContext;
+
+  public MvAppendLongEvaluator(Source source, EvalOperator.ExpressionEvaluator field1,
+      EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) {
+    this.warnings = new Warnings(source);
+    this.field1 = field1;
+    this.field2 = field2;
+    this.driverContext = driverContext;
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (LongBlock field1Block = (LongBlock) field1.eval(page)) {
+      try (LongBlock field2Block = (LongBlock) field2.eval(page)) {
+        return eval(page.getPositionCount(), field1Block, field2Block);
+      }
+    }
+  }
+
+  public LongBlock eval(int positionCount, LongBlock field1Block, LongBlock field2Block) {
+    try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) {
+      position: for (int p = 0; p < positionCount; p++) {
+        boolean allBlocksAreNulls = true;
+        if (!field1Block.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (!field2Block.isNull(p)) {
+          allBlocksAreNulls = false;
+        }
+        if (allBlocksAreNulls) {
+          result.appendNull();
+          continue position;
+        }
+        MvAppend.process(result, p, field1Block, field2Block);
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "MvAppendLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(field1, field2);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field1;
+
+    private final EvalOperator.ExpressionEvaluator.Factory field2;
+
+    public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1,
+        EvalOperator.ExpressionEvaluator.Factory field2) {
+      this.source = source;
+      this.field1 = field1;
+      this.field2 = field2;
+    }
+
+    @Override
+    public MvAppendLongEvaluator get(DriverContext context) {
+      return new MvAppendLongEvaluator(source, field1.get(context), field2.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "MvAppendLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index 675b99c61bfbe..61134da3ecb85 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -27,6 +27,11 @@ public class EsqlCapabilities {
      */
     private static final String FN_CBRT = "fn_cbrt";
 
+    /**
+     * Support for {@code MV_APPEND} function. #107001
+     */
+    private static final String FN_MV_APPEND = "fn_mv_append";
+
     /**
      * Support for function {@code IP_PREFIX}.
      */
@@ -61,6 +66,7 @@ private static Set<String> capabilities() {
         caps.add(FN_SUBSTRING_EMPTY_NULL);
         caps.add(ST_CENTROID_AGG_OPTIMIZED);
         caps.add(METADATA_IGNORED_FIELD);
+        caps.add(FN_MV_APPEND);
 
         if (Build.current().isSnapshot()) {
             caps.add(LOOKUP);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index a8ab961fc201a..af287399a8ddb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -76,6 +76,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tan;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tanh;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tau;
+import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppend;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAvg;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvConcat;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount;
@@ -280,6 +281,7 @@ private FunctionDefinition[][] functions() {
                 def(ToVersion.class, ToVersion::new, "to_version", "to_ver"), },
             // multivalue functions
             new FunctionDefinition[] {
+                def(MvAppend.class, MvAppend::new, "mv_append"),
                 def(MvAvg.class, MvAvg::new, "mv_avg"),
                 def(MvConcat.class, MvConcat::new, "mv_concat"),
                 def(MvCount.class, MvCount::new, "mv_count"),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java
new file mode 100644
index 0000000000000..21da054122aeb
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java
@@ -0,0 +1,285 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.ann.Evaluator;
+import org.elasticsearch.compute.data.BooleanBlock;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+import org.elasticsearch.xpack.esql.planner.PlannerUtils;
+import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+
+/**
+ * Appends values to a multi-value
+ */
+public class MvAppend extends EsqlScalarFunction implements EvaluatorMapper {
+    private final Expression field1, field2;
+    private DataType dataType;
+
+    @FunctionInfo(
+        returnType = {
+            "boolean",
+            "cartesian_point",
+            "cartesian_shape",
+            "date",
+            "double",
+            "geo_point",
+            "geo_shape",
+            "integer",
+            "ip",
+            "keyword",
+            "long",
+            "text",
+            "version" },
+        description = "Concatenates values of two multi-value fields."
+    )
+    public MvAppend(
+        Source source,
+        @Param(
+            name = "field1",
+            type = {
+                "boolean",
+                "cartesian_point",
+                "cartesian_shape",
+                "date",
+                "double",
+                "geo_point",
+                "geo_shape",
+                "integer",
+                "ip",
+                "keyword",
+                "long",
+                "text",
+                "version" }
+        ) Expression field1,
+        @Param(
+            name = "field2",
+            type = {
+                "boolean",
+                "cartesian_point",
+                "cartesian_shape",
+                "date",
+                "double",
+                "geo_point",
+                "geo_shape",
+                "integer",
+                "ip",
+                "keyword",
+                "long",
+                "text",
+                "version" }
+        ) Expression field2
+    ) {
+        super(source, Arrays.asList(field1, field2));
+        this.field1 = field1;
+        this.field2 = field2;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution resolution = isType(field1, EsqlDataTypes::isRepresentable, sourceText(), FIRST, "representable");
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+        dataType = field1.dataType();
+        if (dataType == DataTypes.NULL) {
+            dataType = field2.dataType();
+            return isType(field2, EsqlDataTypes::isRepresentable, sourceText(), SECOND, "representable");
+        }
+        return isType(field2, t -> t == dataType, sourceText(), SECOND, dataType.typeName());
+    }
+
+    @Override
+    public boolean foldable() {
+        return field1.foldable() && field2.foldable();
+    }
+
+    @Override
+    public EvalOperator.ExpressionEvaluator.Factory toEvaluator(
+        Function<Expression, EvalOperator.ExpressionEvaluator.Factory> toEvaluator
+    ) {
+        return switch (PlannerUtils.toElementType(dataType())) {
+            case BOOLEAN -> new MvAppendBooleanEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2));
+            case BYTES_REF -> new MvAppendBytesRefEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2));
+            case DOUBLE -> new MvAppendDoubleEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2));
+            case INT -> new MvAppendIntEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2));
+            case LONG -> new MvAppendLongEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2));
+            case NULL -> EvalOperator.CONSTANT_NULL_FACTORY;
+            default -> throw EsqlIllegalArgumentException.illegalDataType(dataType);
+        };
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new MvAppend(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, MvAppend::new, field1, field2);
+    }
+
+    @Override
+    public DataType dataType() {
+        if (dataType == null) {
+            resolveType();
+        }
+        return dataType;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field1, field2);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+        MvAppend other = (MvAppend) obj;
+        return Objects.equals(other.field1, field1) && Objects.equals(other.field2, field2);
+    }
+
+    @Evaluator(extraName = "Int")
+    static void process(IntBlock.Builder builder, int position, IntBlock field1, IntBlock field2) {
+        int count1 = field1.getValueCount(position);
+        int count2 = field2.getValueCount(position);
+        if (count1 == 0 || count2 == 0) {
+            builder.appendNull();
+        } else {
+            builder.beginPositionEntry();
+            int first1 = field1.getFirstValueIndex(position);
+            int first2 = field2.getFirstValueIndex(position);
+            for (int i = 0; i < count1; i++) {
+                builder.appendInt(field1.getInt(first1 + i));
+            }
+            for (int i = 0; i < count2; i++) {
+                builder.appendInt(field2.getInt(first2 + i));
+            }
+            builder.endPositionEntry();
+        }
+
+    }
+
+    @Evaluator(extraName = "Boolean")
+    static void process(BooleanBlock.Builder builder, int position, BooleanBlock field1, BooleanBlock field2) {
+        int count1 = field1.getValueCount(position);
+        int count2 = field2.getValueCount(position);
+        if (count1 == 0 || count2 == 0) {
+            builder.appendNull();
+        } else {
+            int first1 = field1.getFirstValueIndex(position);
+            int first2 = field2.getFirstValueIndex(position);
+            builder.beginPositionEntry();
+            for (int i = 0; i < count1; i++) {
+                builder.appendBoolean(field1.getBoolean(first1 + i));
+            }
+            for (int i = 0; i < count2; i++) {
+                builder.appendBoolean(field2.getBoolean(first2 + i));
+            }
+            builder.endPositionEntry();
+        }
+
+    }
+
+    @Evaluator(extraName = "Long")
+    static void process(LongBlock.Builder builder, int position, LongBlock field1, LongBlock field2) {
+        int count1 = field1.getValueCount(position);
+        int count2 = field2.getValueCount(position);
+        if (count1 == 0 || count2 == 0) {
+            builder.appendNull();
+        } else {
+            int first1 = field1.getFirstValueIndex(position);
+            int first2 = field2.getFirstValueIndex(position);
+            builder.beginPositionEntry();
+            for (int i = 0; i < count1; i++) {
+                builder.appendLong(field1.getLong(first1 + i));
+            }
+            for (int i = 0; i < count2; i++) {
+                builder.appendLong(field2.getLong(first2 + i));
+            }
+            builder.endPositionEntry();
+        }
+    }
+
+    @Evaluator(extraName = "Double")
+    static void process(DoubleBlock.Builder builder, int position, DoubleBlock field1, DoubleBlock field2) {
+        int count1 = field1.getValueCount(position);
+        int count2 = field2.getValueCount(position);
+        if (count1 == 0 || count2 == 0) {
+            builder.appendNull();
+        } else {
+            int first1 = field1.getFirstValueIndex(position);
+            int first2 = field2.getFirstValueIndex(position);
+            builder.beginPositionEntry();
+            for (int i = 0; i < count1; i++) {
+                builder.appendDouble(field1.getDouble(first1 + i));
+            }
+            for (int i = 0; i < count2; i++) {
+                builder.appendDouble(field2.getDouble(first2 + i));
+            }
+            builder.endPositionEntry();
+        }
+
+    }
+
+    @Evaluator(extraName = "BytesRef")
+    static void process(BytesRefBlock.Builder builder, int position, BytesRefBlock field1, BytesRefBlock field2) {
+        int count1 = field1.getValueCount(position);
+        int count2 = field2.getValueCount(position);
+        if (count1 == 0 || count2 == 0) {
+            builder.appendNull();
+        } else {
+            int first1 = field1.getFirstValueIndex(position);
+            int first2 = field2.getFirstValueIndex(position);
+            builder.beginPositionEntry();
+            BytesRef spare = new BytesRef();
+            for (int i = 0; i < count1; i++) {
+                builder.appendBytesRef(field1.getBytesRef(first1 + i, spare));
+            }
+            for (int i = 0; i < count2; i++) {
+                builder.appendBytesRef(field2.getBytesRef(first2 + i, spare));
+            }
+            builder.endPositionEntry();
+        }
+    }
+
+    @Override
+    public Nullability nullable() {
+        return Nullability.TRUE;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index f605f898366e1..6b5e91376a335 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -118,6 +118,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tanh;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tau;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.AbstractMultivalueFunction;
+import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppend;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAvg;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvConcat;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount;
@@ -425,6 +426,7 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(AggregateFunction.class, Sum.class, PlanNamedTypes::writeAggFunction, PlanNamedTypes::readAggFunction),
             of(AggregateFunction.class, Values.class, PlanNamedTypes::writeAggFunction, PlanNamedTypes::readAggFunction),
             // Multivalue functions
+            of(ScalarFunction.class, MvAppend.class, PlanNamedTypes::writeMvAppend, PlanNamedTypes::readMvAppend),
             of(ScalarFunction.class, MvAvg.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction),
             of(ScalarFunction.class, MvCount.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction),
             of(ScalarFunction.class, MvConcat.class, PlanNamedTypes::writeMvConcat, PlanNamedTypes::readMvConcat),
@@ -1857,4 +1859,16 @@ static void writeMvZip(PlanStreamOutput out, MvZip fn) throws IOException {
         out.writeExpression(fields.get(1));
         out.writeOptionalWriteable(fields.size() == 3 ? o -> out.writeExpression(fields.get(2)) : null);
     }
+
+    static MvAppend readMvAppend(PlanStreamInput in) throws IOException {
+        return new MvAppend(Source.readFrom(in), in.readExpression(), in.readExpression());
+    }
+
+    static void writeMvAppend(PlanStreamOutput out, MvAppend fn) throws IOException {
+        Source.EMPTY.writeTo(out);
+        List<Expression> fields = fn.children();
+        assert fields.size() == 2;
+        out.writeExpression(fields.get(0));
+        out.writeExpression(fields.get(1));
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index 5f2b4290f48f3..2d9b52f1b433f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -62,6 +62,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -522,7 +523,9 @@ public void testDropUnsupportedFieldExplicit() {
             "float",
             "foo_type",
             "int",
+            "ip",
             "keyword",
+            "long",
             "point",
             "shape",
             "some.ambiguous",
@@ -566,7 +569,9 @@ public void testDropUnsupportedPattern() {
             "float",
             "foo_type",
             "int",
+            "ip",
             "keyword",
+            "long",
             "point",
             "shape",
             "some.ambiguous",
@@ -779,7 +784,9 @@ public void testDropSupportedDottedField() {
             "float",
             "foo_type",
             "int",
+            "ip",
             "keyword",
+            "long",
             "point",
             "shape",
             "some.ambiguous",
@@ -1878,6 +1885,40 @@ public void testInOnText() {
             """, "mapping-multi-field-variation.json", "text");
     }
 
+    public void testMvAppendValidation() {
+        String[][] fields = {
+            { "bool", "boolean" },
+            { "int", "integer" },
+            { "unsigned_long", "unsigned_long" },
+            { "float", "double" },
+            { "text", "text" },
+            { "keyword", "keyword" },
+            { "date", "datetime" },
+            { "point", "geo_point" },
+            { "shape", "geo_shape" },
+            { "long", "long" },
+            { "ip", "ip" },
+            { "version", "version" } };
+
+        Supplier<Integer> supplier = () -> randomInt(fields.length - 1);
+        int first = supplier.get();
+        int second = randomValueOtherThan(first, supplier);
+
+        String signature = "mv_append(" + fields[first][0] + ", " + fields[second][0] + ")";
+        verifyUnsupported(
+            " from test | eval " + signature,
+            "second argument of ["
+                + signature
+                + "] must be ["
+                + fields[first][1]
+                + "], found value ["
+                + fields[second][0]
+                + "] type ["
+                + fields[second][1]
+                + "]"
+        );
+    }
+
     public void testLookup() {
         var e = expectThrows(ParsingException.class, () -> analyze("""
               FROM test
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
new file mode 100644
index 0000000000000..6d43a1087ad88
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
@@ -0,0 +1,296 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geo.ShapeTestUtils;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
+import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
+import static org.hamcrest.Matchers.equalTo;
+
+public class MvAppendTests extends AbstractFunctionTestCase {
+    public MvAppendTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        List<TestCaseSupplier> suppliers = new ArrayList<>();
+        booleans(suppliers);
+        ints(suppliers);
+        longs(suppliers);
+        doubles(suppliers);
+        bytesRefs(suppliers);
+        nulls(suppliers);
+        return parameterSuppliersFromTypedData(suppliers);
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new MvAppend(source, args.get(0), args.get(1));
+    }
+
+    private static void booleans(List<TestCaseSupplier> suppliers) {
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.BOOLEAN, DataTypes.BOOLEAN), () -> {
+            List<Boolean> field1 = randomList(1, 10, () -> randomBoolean());
+            List<Boolean> field2 = randomList(1, 10, () -> randomBoolean());
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.BOOLEAN, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.BOOLEAN, "field2")
+                ),
+                "MvAppendBooleanEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.BOOLEAN,
+                equalTo(result)
+            );
+        }));
+    }
+
+    private static void ints(List<TestCaseSupplier> suppliers) {
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+            List<Integer> field1 = randomList(1, 10, () -> randomInt());
+            List<Integer> field2 = randomList(1, 10, () -> randomInt());
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.INTEGER, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.INTEGER, "field2")
+                ),
+                "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.INTEGER,
+                equalTo(result)
+            );
+        }));
+    }
+
+    private static void longs(List<TestCaseSupplier> suppliers) {
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.LONG, DataTypes.LONG), () -> {
+            List<Long> field1 = randomList(1, 10, () -> randomLong());
+            List<Long> field2 = randomList(1, 10, () -> randomLong());
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.LONG, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.LONG, "field2")
+                ),
+                "MvAppendLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.LONG,
+                equalTo(result)
+            );
+        }));
+
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.DATETIME, DataTypes.DATETIME), () -> {
+            List<Long> field1 = randomList(1, 10, () -> randomLong());
+            List<Long> field2 = randomList(1, 10, () -> randomLong());
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.DATETIME, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.DATETIME, "field2")
+                ),
+                "MvAppendLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.DATETIME,
+                equalTo(result)
+            );
+        }));
+    }
+
+    private static void doubles(List<TestCaseSupplier> suppliers) {
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.DOUBLE, DataTypes.DOUBLE), () -> {
+            List<Double> field1 = randomList(1, 10, () -> randomDouble());
+            List<Double> field2 = randomList(1, 10, () -> randomDouble());
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.DOUBLE, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.DOUBLE, "field2")
+                ),
+                "MvAppendDoubleEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.DOUBLE,
+                equalTo(result)
+            );
+        }));
+    }
+
+    private static void bytesRefs(List<TestCaseSupplier> suppliers) {
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.KEYWORD, DataTypes.KEYWORD), () -> {
+            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataTypes.KEYWORD).value());
+            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataTypes.KEYWORD).value());
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.KEYWORD, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.KEYWORD, "field2")
+                ),
+                "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.KEYWORD,
+                equalTo(result)
+            );
+        }));
+
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.TEXT, DataTypes.TEXT), () -> {
+            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataTypes.TEXT).value());
+            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataTypes.TEXT).value());
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.TEXT, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.TEXT, "field2")
+                ),
+                "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.TEXT,
+                equalTo(result)
+            );
+        }));
+
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.IP), () -> {
+            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataTypes.IP).value());
+            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataTypes.IP).value());
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.IP, "field"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.IP, "field")
+                ),
+                "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.IP,
+                equalTo(result)
+            );
+        }));
+
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.VERSION, DataTypes.VERSION), () -> {
+            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataTypes.VERSION).value());
+            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataTypes.VERSION).value());
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.VERSION, "field"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.VERSION, "field")
+                ),
+                "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.VERSION,
+                equalTo(result)
+            );
+        }));
+
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.GEO_POINT, DataTypes.GEO_POINT), () -> {
+            List<Object> field1 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint())));
+            List<Object> field2 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint())));
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.GEO_POINT, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.GEO_POINT, "field2")
+                ),
+                "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.GEO_POINT,
+                equalTo(result)
+            );
+        }));
+
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_POINT), () -> {
+            List<Object> field1 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint())));
+            List<Object> field2 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint())));
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.CARTESIAN_POINT, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.CARTESIAN_POINT, "field2")
+                ),
+                "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.CARTESIAN_POINT,
+                equalTo(result)
+            );
+        }));
+
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.GEO_SHAPE, DataTypes.GEO_SHAPE), () -> {
+            List<Object> field1 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))));
+            List<Object> field2 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))));
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.GEO_SHAPE, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.GEO_SHAPE, "field2")
+                ),
+                "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.GEO_SHAPE,
+                equalTo(result)
+            );
+        }));
+
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.CARTESIAN_SHAPE, DataTypes.CARTESIAN_SHAPE), () -> {
+            List<Object> field1 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))));
+            List<Object> field2 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))));
+            var result = new ArrayList<>(field1);
+            result.addAll(field2);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.CARTESIAN_SHAPE, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.CARTESIAN_SHAPE, "field2")
+                ),
+                "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.CARTESIAN_SHAPE,
+                equalTo(result)
+            );
+        }));
+    }
+
+    private static void nulls(List<TestCaseSupplier> suppliers) {
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+            List<Integer> field2 = randomList(2, 10, () -> randomInt());
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(null, DataTypes.INTEGER, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataTypes.INTEGER, "field2")
+                ),
+                "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.INTEGER,
+                equalTo(null)
+            );
+        }));
+        suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+            List<Integer> field1 = randomList(2, 10, () -> randomInt());
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(field1, DataTypes.INTEGER, "field1"),
+                    new TestCaseSupplier.TypedData(null, DataTypes.INTEGER, "field2")
+                ),
+                "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
+                DataTypes.INTEGER,
+                equalTo(null)
+            );
+        }));
+    }
+}

From 5f97474c57fb1e1b5914eb05f712b78d23200d4b Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Wed, 5 Jun 2024 04:29:41 +1000
Subject: [PATCH 139/208] Mute
 org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
 org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT #109362

---
 muted-tests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 628c57301f3bf..7eb9275a571c5 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -19,8 +19,7 @@ tests:
   method: "testGuessIsDayFirstFromLocale"
 - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108857"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
-    \ dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
 - class: "org.elasticsearch.upgrades.SearchStatesIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108991"
   method: "testCanMatch"
@@ -29,8 +28,7 @@ tests:
   method: "testTrainedModelInference"
 - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109188"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
-    \ dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
 - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109189"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
@@ -60,6 +58,8 @@ tests:
 - class: "org.elasticsearch.client.RestClientSingleHostIntegTests"
   issue: "https://github.com/elastic/elasticsearch/issues/102717"
   method: "testRequestResetAndAbort"
+- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
+  issue: https://github.com/elastic/elasticsearch/issues/109362
 
 # Examples:
 #

From 68c9ba1f6efdb3968b478656f3e0ad459c0a6fd1 Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Wed, 5 Jun 2024 04:42:27 +1000
Subject: [PATCH 140/208] Mute
 org.elasticsearch.upgrades.UpgradeClusterClientYamlTestSuiteIT
 org.elasticsearch.upgrades.UpgradeClusterClientYamlTestSuiteIT #109364

---
 muted-tests.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index 7eb9275a571c5..373c3e8cc65c4 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -60,6 +60,8 @@ tests:
   method: "testRequestResetAndAbort"
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/109362
+- class: org.elasticsearch.upgrades.UpgradeClusterClientYamlTestSuiteIT
+  issue: https://github.com/elastic/elasticsearch/issues/109364
 
 # Examples:
 #

From 16849c3b9fa5ea59e93cf8b464c3b1776361c4c6 Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Tue, 4 Jun 2024 15:08:51 -0400
Subject: [PATCH 141/208] [ESQL] Make Datatypes an enum (#109227)

After #109162, all instances of DataType were in one class, as a collection of static constants. That pattern can then be simplified into an enum, merging the behavior class and the static collection class. That opens the door to future optimizations, like using enum serialization rather than string serialization to save bytes over the wire. It also makes the code easier to read, as all the behavior is now in a single file (which is still pretty short).

Most of this PR is just juggling names around to have the two references refer to the same thing.

Follow up work can merge the functions from EsqlDataTypes into this enum, but this PR is long enough already.
---
 .../compute/operator/EvalBenchmark.java       |  16 +-
 .../esql/core/analyzer/VerifierChecks.java    |   2 +-
 .../extractor/AbstractFieldHitExtractor.java  |   3 +-
 .../esql/core/expression/EmptyAttribute.java  |   3 +-
 .../esql/core/expression/Expressions.java     |   4 +-
 .../xpack/esql/core/expression/Literal.java   |   7 +-
 .../core/expression/MetadataAttribute.java    |  11 +-
 .../esql/core/expression/TypeResolutions.java |  11 +-
 ...naryComparisonCaseInsensitiveFunction.java |   3 +-
 .../whitelist/InternalQlScriptUtils.java      |   2 +-
 .../esql/core/expression/predicate/Range.java |   5 +-
 .../predicate/fulltext/FullTextPredicate.java |   3 +-
 .../predicate/logical/BinaryLogic.java        |   3 +-
 .../expression/predicate/logical/Not.java     |   5 +-
 .../expression/predicate/nulls/IsNotNull.java |   5 +-
 .../expression/predicate/nulls/IsNull.java    |   5 +-
 .../predicate/operator/arithmetic/Mul.java    |   3 +-
 .../operator/comparison/BinaryComparison.java |   3 +-
 .../predicate/operator/comparison/In.java     |   5 +-
 .../predicate/regex/RegexMatch.java           |   3 +-
 .../esql/core/index/IndexCompatibility.java   |   6 +-
 .../xpack/esql/core/index/IndexResolver.java  |  10 +-
 .../index/VersionCompatibilityChecks.java     |   4 +-
 .../esql/core/optimizer/OptimizerRules.java   |  40 +--
 .../core/planner/ExpressionTranslators.java   |  13 +-
 .../xpack/esql/core/type/DataType.java        | 241 ++++++++++++----
 .../esql/core/type/DataTypeConverter.java     |  36 +--
 .../xpack/esql/core/type/DataTypes.java       | 217 ---------------
 .../xpack/esql/core/type/DateEsField.java     |   4 +-
 .../core/type/DefaultDataTypeRegistry.java    |   8 +-
 .../esql/core/type/InvalidMappedField.java    |   2 +-
 .../xpack/esql/core/type/KeywordEsField.java  |   2 +-
 .../xpack/esql/core/type/TextEsField.java     |   4 +-
 .../xpack/esql/core/type/Types.java           |  12 +-
 .../esql/core/type/UnsupportedEsField.java    |   2 +-
 .../core/expression/AttributeMapTests.java    |  20 +-
 .../esql/core/expression/LiteralTests.java    |  16 +-
 .../core/expression/NullabilityTests.java     |   3 +-
 .../function/scalar/FunctionTestUtils.java    |  11 +-
 .../core/expression/predicate/RangeTests.java |  18 +-
 .../core/optimizer/OptimizerRulesTests.java   |  11 +-
 .../core/querydsl/query/MatchQueryTests.java  |   2 +-
 .../core/type/DataTypeConversionTests.java    |  34 +--
 .../xpack/esql/core/type/EsFieldTests.java    |   4 +-
 .../xpack/esql/core/type/TypesTests.java      |  20 +-
 .../xpack/esql/core/TestUtils.java            |   7 +-
 .../xpack/esql/EsqlTestUtils.java             |   5 +-
 .../function/scalar/ip/IpPrefixEvaluator.java |   2 +-
 .../xpack/esql/action/ParseTables.java        |   9 +-
 .../xpack/esql/analysis/Analyzer.java         |  37 ++-
 .../xpack/esql/analysis/Verifier.java         |  31 +--
 .../xpack/esql/enrich/QueryList.java          |   4 +-
 .../esql/expression/EsqlTypeResolutions.java  |  13 +-
 .../xpack/esql/expression/Order.java          |   4 +-
 .../function/EsqlFunctionRegistry.java        |  33 ++-
 .../expression/function/aggregate/Avg.java    |   5 +-
 .../expression/function/aggregate/Count.java  |   9 +-
 .../function/aggregate/CountDistinct.java     |  17 +-
 .../expression/function/aggregate/Median.java |   7 +-
 .../function/aggregate/NumericAggregate.java  |  15 +-
 .../function/aggregate/Percentile.java        |   4 +-
 .../function/aggregate/SpatialCentroid.java   |   9 +-
 .../expression/function/aggregate/Sum.java    |   9 +-
 .../expression/function/aggregate/Values.java |  11 +-
 .../expression/function/grouping/Bucket.java  |  15 +-
 .../function/scalar/conditional/Case.java     |   2 +-
 .../function/scalar/conditional/Greatest.java |  21 +-
 .../function/scalar/conditional/Least.java    |  21 +-
 .../convert/AbstractConvertFunction.java      |  12 +-
 .../function/scalar/convert/FromBase64.java   |   2 +-
 .../function/scalar/convert/ToBase64.java     |   2 +-
 .../function/scalar/convert/ToBoolean.java    |  14 +-
 .../scalar/convert/ToCartesianPoint.java      |   6 +-
 .../scalar/convert/ToCartesianShape.java      |   8 +-
 .../function/scalar/convert/ToDatetime.java   |  14 +-
 .../function/scalar/convert/ToDegrees.java    |   8 +-
 .../function/scalar/convert/ToDouble.java     |  23 +-
 .../function/scalar/convert/ToGeoPoint.java   |   6 +-
 .../function/scalar/convert/ToGeoShape.java   |   8 +-
 .../function/scalar/convert/ToIP.java         |   6 +-
 .../function/scalar/convert/ToInteger.java    |  19 +-
 .../function/scalar/convert/ToLong.java       |  21 +-
 .../function/scalar/convert/ToRadians.java    |   8 +-
 .../function/scalar/convert/ToString.java     |  28 +-
 .../scalar/convert/ToUnsignedLong.java        |  16 +-
 .../function/scalar/convert/ToVersion.java    |   6 +-
 .../scalar/date/BinaryDateTimeFunction.java   |   3 +-
 .../function/scalar/date/DateDiff.java        |   3 +-
 .../function/scalar/date/DateExtract.java     |   3 +-
 .../function/scalar/date/DateFormat.java      |   3 +-
 .../function/scalar/date/DateParse.java       |   3 +-
 .../function/scalar/date/DateTrunc.java       |   3 +-
 .../expression/function/scalar/date/Now.java  |   3 +-
 .../function/scalar/ip/CIDRMatch.java         |   3 +-
 .../function/scalar/ip/IpPrefix.java          |   5 +-
 .../expression/function/scalar/math/Abs.java  |  10 +-
 .../math/AbstractTrigonometricFunction.java   |   5 +-
 .../function/scalar/math/Atan2.java           |   7 +-
 .../expression/function/scalar/math/Cast.java |  21 +-
 .../expression/function/scalar/math/Cbrt.java |  11 +-
 .../scalar/math/DoubleConstantFunction.java   |   3 +-
 .../expression/function/scalar/math/Log.java  |   7 +-
 .../function/scalar/math/Log10.java           |  11 +-
 .../expression/function/scalar/math/Pow.java  |   7 +-
 .../function/scalar/math/Round.java           |  11 +-
 .../function/scalar/math/Signum.java          |  11 +-
 .../expression/function/scalar/math/Sqrt.java |  11 +-
 .../function/scalar/multivalue/MvAvg.java     |   5 +-
 .../function/scalar/multivalue/MvConcat.java  |   3 +-
 .../function/scalar/multivalue/MvCount.java   |   3 +-
 .../function/scalar/multivalue/MvMedian.java  |   4 +-
 .../function/scalar/multivalue/MvSlice.java   |   2 +-
 .../function/scalar/multivalue/MvSort.java    |   3 +-
 .../function/scalar/multivalue/MvSum.java     |   4 +-
 .../function/scalar/multivalue/MvZip.java     |   5 +-
 .../function/scalar/nulls/Coalesce.java       |   2 +-
 .../scalar/spatial/SpatialContains.java       |   8 +-
 .../scalar/spatial/SpatialDisjoint.java       |   8 +-
 .../scalar/spatial/SpatialIntersects.java     |   8 +-
 .../spatial/SpatialRelatesFunction.java       |   9 +-
 .../scalar/spatial/SpatialWithin.java         |   8 +-
 .../function/scalar/spatial/StX.java          |   2 +-
 .../function/scalar/spatial/StY.java          |   2 +-
 .../function/scalar/string/Concat.java        |   3 +-
 .../function/scalar/string/EndsWith.java      |   3 +-
 .../function/scalar/string/Left.java          |   5 +-
 .../function/scalar/string/Length.java        |   3 +-
 .../function/scalar/string/Locate.java        |   5 +-
 .../function/scalar/string/Replace.java       |   5 +-
 .../function/scalar/string/Right.java         |   5 +-
 .../function/scalar/string/Split.java         |   3 +-
 .../function/scalar/string/StartsWith.java    |   3 +-
 .../function/scalar/string/Substring.java     |   5 +-
 .../DateTimeArithmeticOperation.java          |  17 +-
 .../arithmetic/EsqlArithmeticOperation.java   |  13 +-
 .../predicate/operator/arithmetic/Neg.java    |  13 +-
 .../predicate/operator/arithmetic/Sub.java    |   4 +-
 .../predicate/operator/comparison/Equals.java |  29 +-
 .../comparison/EsqlBinaryComparison.java      |  13 +-
 .../operator/comparison/GreaterThan.java      |  19 +-
 .../comparison/GreaterThanOrEqual.java        |  19 +-
 .../predicate/operator/comparison/In.java     |   3 +-
 .../InsensitiveBinaryComparison.java          |   3 +-
 .../comparison/InsensitiveEqualsMapper.java   |   3 +-
 .../operator/comparison/LessThan.java         |  19 +-
 .../operator/comparison/LessThanOrEqual.java  |  19 +-
 .../operator/comparison/NotEquals.java        |  29 +-
 .../xpack/esql/io/stream/PlanNamedTypes.java  |   4 +-
 .../optimizer/LocalLogicalPlanOptimizer.java  |   5 +-
 .../optimizer/LocalPhysicalPlanOptimizer.java |   6 +-
 .../esql/optimizer/LogicalPlanOptimizer.java  |   6 +-
 .../xpack/esql/optimizer/OptimizerRules.java  |  16 +-
 .../rules/SimplifyComparisonsArithmetics.java |   3 +-
 .../xpack/esql/parser/ExpressionBuilder.java  |  51 ++--
 .../xpack/esql/parser/LogicalPlanBuilder.java |  10 +-
 .../xpack/esql/plan/logical/Explain.java      |   6 +-
 .../xpack/esql/plan/logical/Grok.java         |  13 +-
 .../esql/plan/logical/meta/MetaFunctions.java |   4 +-
 .../esql/plan/logical/show/ShowInfo.java      |   2 +-
 .../xpack/esql/plan/physical/EsQueryExec.java |  10 +-
 .../xpack/esql/planner/AggregateMapper.java   |  31 +--
 .../planner/EsqlExpressionTranslators.java    |  25 +-
 .../xpack/esql/planner/PlannerUtils.java      |  33 ++-
 .../querydsl/query/SpatialRelatesQuery.java   |   2 +-
 .../xpack/esql/session/EsqlIndexResolver.java |  10 +-
 .../xpack/esql/session/EsqlSession.java       |   4 +-
 .../esql/type/EsqlDataTypeConverter.java      |  69 +++--
 .../xpack/esql/type/EsqlDataTypeRegistry.java |  11 +-
 .../xpack/esql/type/EsqlDataTypes.java        |  67 +++--
 .../elasticsearch/xpack/esql/CsvTests.java    |   2 +-
 .../esql/action/EsqlQueryRequestTests.java    |  24 +-
 .../esql/action/EsqlQueryResponseTests.java   |   5 +-
 .../xpack/esql/analysis/AnalyzerTests.java    |  35 ++-
 .../xpack/esql/analysis/VerifierTests.java    |   7 +-
 .../EnrichQuerySourceOperatorTests.java       |   2 +-
 .../xpack/esql/expression/CanonicalTests.java |  11 +-
 .../function/AbstractFunctionTestCase.java    | 257 +++++++++---------
 .../function/FieldAttributeTests.java         |   5 +-
 .../function/MetadataAttributeTests.java      |   5 +-
 .../function/ReferenceAttributeTests.java     |   5 +-
 .../expression/function/TestCaseSupplier.java | 203 +++++++-------
 .../AbstractScalarFunctionTestCase.java       |  29 +-
 .../function/scalar/NamedExpressionTests.java |   4 +-
 .../scalar/VaragsTestCaseBuilder.java         |  43 ++-
 .../scalar/conditional/CaseExtraTests.java    |  92 +++----
 .../scalar/conditional/CaseTests.java         | 119 ++++----
 .../scalar/conditional/GreatestTests.java     |  34 +--
 .../scalar/conditional/LeastTests.java        |  34 +--
 .../scalar/convert/FromBase64Tests.java       |  14 +-
 .../scalar/convert/ToBase64Tests.java         |  18 +-
 .../scalar/convert/ToBooleanTests.java        |  14 +-
 .../scalar/convert/ToCartesianPointTests.java |   9 +-
 .../scalar/convert/ToCartesianShapeTests.java |  11 +-
 .../scalar/convert/ToDatetimeTests.java       |  30 +-
 .../scalar/convert/ToDegreesTests.java        |  20 +-
 .../scalar/convert/ToDoubleTests.java         |  32 +--
 .../scalar/convert/ToGeoPointTests.java       |   9 +-
 .../scalar/convert/ToGeoShapeTests.java       |  11 +-
 .../function/scalar/convert/ToIPTests.java    |  14 +-
 .../scalar/convert/ToIntegerTests.java        |  54 ++--
 .../function/scalar/convert/ToLongTests.java  |  46 ++--
 .../scalar/convert/ToRadiansTests.java        |  10 +-
 .../scalar/convert/ToStringTests.java         |  28 +-
 .../scalar/convert/ToUnsignedLongTests.java   |  40 +--
 .../scalar/convert/ToVersionTests.java        |   7 +-
 .../function/scalar/date/DateDiffTests.java   |  46 ++--
 .../scalar/date/DateExtractTests.java         |  34 +--
 .../function/scalar/date/DateFormatTests.java |  18 +-
 .../function/scalar/date/DateParseTests.java  |  54 ++--
 .../function/scalar/date/DateTruncTests.java  |  26 +-
 .../function/scalar/ip/CIDRMatchTests.java    |  34 +--
 .../function/scalar/ip/IpPrefixTests.java     |  42 +--
 .../function/scalar/math/AbsTests.java        |  21 +-
 .../function/scalar/math/BucketTests.java     |  67 +++--
 .../function/scalar/math/CbrtTests.java       |  10 +-
 .../function/scalar/math/CeilTests.java       |  15 +-
 .../function/scalar/math/ETests.java          |   6 +-
 .../function/scalar/math/FloorTests.java      |  10 +-
 .../function/scalar/math/Log10Tests.java      |  18 +-
 .../function/scalar/math/LogTests.java        |  29 +-
 .../function/scalar/math/NowTests.java        |   4 +-
 .../function/scalar/math/PiTests.java         |   6 +-
 .../function/scalar/math/PowTests.java        |   7 +-
 .../function/scalar/math/RoundTests.java      |  36 ++-
 .../function/scalar/math/SignumTests.java     |  10 +-
 .../function/scalar/math/SqrtTests.java       |  16 +-
 .../function/scalar/math/TauTests.java        |   6 +-
 .../AbstractMultivalueFunctionTestCase.java   | 103 ++++---
 .../scalar/multivalue/MvAvgTests.java         |  11 +-
 .../scalar/multivalue/MvConcatTests.java      |   7 +-
 .../scalar/multivalue/MvCountTests.java       |  25 +-
 .../scalar/multivalue/MvDedupeTests.java      |   5 +-
 .../scalar/multivalue/MvFirstTests.java       |  21 +-
 .../scalar/multivalue/MvLastTests.java        |  21 +-
 .../scalar/multivalue/MvMedianTests.java      |  13 +-
 .../scalar/multivalue/MvSliceTests.java       | 174 ++++++------
 .../scalar/multivalue/MvSortTests.java        |  82 +++---
 .../scalar/multivalue/MvSumTests.java         |  11 +-
 .../scalar/multivalue/MvZipTests.java         |  31 +--
 .../function/scalar/nulls/CoalesceTests.java  |  19 +-
 .../function/scalar/nulls/IsNotNullTests.java |   9 +-
 .../function/scalar/nulls/IsNullTests.java    |   9 +-
 .../scalar/spatial/SpatialContainsTests.java  |   5 +-
 .../scalar/spatial/SpatialDisjointTests.java  |   5 +-
 .../spatial/SpatialIntersectsTests.java       |   5 +-
 .../SpatialRelatesFunctionTestCase.java       |   3 +-
 .../scalar/spatial/SpatialWithinTests.java    |   5 +-
 .../function/scalar/spatial/StXTests.java     |   2 +-
 .../function/scalar/spatial/StYTests.java     |   2 +-
 .../function/scalar/string/ConcatTests.java   |  29 +-
 .../function/scalar/string/EndsWithTests.java |  38 +--
 .../function/scalar/string/LeftTests.java     |  84 +++---
 .../function/scalar/string/LengthTests.java   |  20 +-
 .../function/scalar/string/LocateTests.java   |  13 +-
 .../function/scalar/string/RLikeTests.java    |  29 +-
 .../function/scalar/string/ReplaceTests.java  |  29 +-
 .../function/scalar/string/RightTests.java    |  84 +++---
 .../function/scalar/string/SplitTests.java    |  11 +-
 .../scalar/string/StartsWithTests.java        |  14 +-
 .../scalar/string/SubstringTests.java         |  86 +++---
 .../function/scalar/string/ToLowerTests.java  |  11 +-
 .../function/scalar/string/ToUpperTests.java  |  11 +-
 .../scalar/string/WildcardLikeTests.java      |   5 +-
 .../AbstractBinaryOperatorTestCase.java       |  15 +-
 .../predicate/operator/BreakerTests.java      |   6 +-
 .../AbstractArithmeticTestCase.java           |  37 ++-
 .../AbstractDateTimeArithmeticTestCase.java   |   7 +-
 .../operator/arithmetic/AddTests.java         |  35 ++-
 .../operator/arithmetic/DivTests.java         |   9 +-
 .../operator/arithmetic/ModTests.java         |   7 +-
 .../operator/arithmetic/MulTests.java         |  16 +-
 .../operator/arithmetic/NegTests.java         |  41 ++-
 .../operator/arithmetic/SubTests.java         |  60 ++--
 .../operator/comparison/EqualsTests.java      |  22 +-
 .../comparison/GreaterThanOrEqualTests.java   |  12 +-
 .../operator/comparison/GreaterThanTests.java |  12 +-
 .../comparison/LessThanOrEqualTests.java      |  12 +-
 .../operator/comparison/LessThanTests.java    |  12 +-
 .../operator/comparison/NotEqualsTests.java   |  22 +-
 .../esql/io/stream/PlanNamedTypesTests.java   |  27 +-
 .../esql/io/stream/PlanStreamOutputTests.java |   4 +-
 .../LocalLogicalPlanOptimizerTests.java       |   8 +-
 .../LocalPhysicalPlanOptimizerTests.java      |   8 +-
 .../optimizer/LogicalPlanOptimizerTests.java  |  31 +--
 .../esql/optimizer/OptimizerRulesTests.java   |  24 +-
 .../optimizer/PhysicalPlanOptimizerTests.java |  31 +--
 .../xpack/esql/parser/ExpressionTests.java    |  12 +-
 .../esql/parser/StatementParserTests.java     |  27 +-
 .../xpack/esql/plan/QueryPlanTests.java       |   2 +-
 .../xpack/esql/planner/EvalMapperTests.java   |  15 +-
 .../planner/LocalExecutionPlannerTests.java   |   6 +-
 .../esql/tree/EsqlNodeSubclassTests.java      |   3 +-
 .../esql/type/EsqlDataTypeRegistryTests.java  |  11 +-
 293 files changed, 2650 insertions(+), 2904 deletions(-)
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
index 50491da4d39aa..7aedbfa9ee996 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
@@ -101,7 +101,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
             case "add" -> {
                 FieldAttribute longField = longField();
                 yield EvalMapper.toEvaluator(
-                    new Add(Source.EMPTY, longField, new Literal(Source.EMPTY, 1L, DataTypes.LONG)),
+                    new Add(Source.EMPTY, longField, new Literal(Source.EMPTY, 1L, DataType.LONG)),
                     layout(longField)
                 ).get(driverContext);
             }
@@ -109,17 +109,17 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 FieldAttribute timestamp = new FieldAttribute(
                     Source.EMPTY,
                     "timestamp",
-                    new EsField("timestamp", DataTypes.DATETIME, Map.of(), true)
+                    new EsField("timestamp", DataType.DATETIME, Map.of(), true)
                 );
                 yield EvalMapper.toEvaluator(
-                    new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataTypes.TIME_DURATION), timestamp),
+                    new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp),
                     layout(timestamp)
                 ).get(driverContext);
             }
             case "equal_to_const" -> {
                 FieldAttribute longField = longField();
                 yield EvalMapper.toEvaluator(
-                    new Equals(Source.EMPTY, longField, new Literal(Source.EMPTY, 100_000L, DataTypes.LONG)),
+                    new Equals(Source.EMPTY, longField, new Literal(Source.EMPTY, 100_000L, DataType.LONG)),
                     layout(longField)
                 ).get(driverContext);
             }
@@ -147,15 +147,15 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
     }
 
     private static FieldAttribute longField() {
-        return new FieldAttribute(Source.EMPTY, "long", new EsField("long", DataTypes.LONG, Map.of(), true));
+        return new FieldAttribute(Source.EMPTY, "long", new EsField("long", DataType.LONG, Map.of(), true));
     }
 
     private static FieldAttribute intField() {
-        return new FieldAttribute(Source.EMPTY, "int", new EsField("int", DataTypes.INTEGER, Map.of(), true));
+        return new FieldAttribute(Source.EMPTY, "int", new EsField("int", DataType.INTEGER, Map.of(), true));
     }
 
     private static FieldAttribute keywordField() {
-        return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataTypes.KEYWORD, Map.of(), true));
+        return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataType.KEYWORD, Map.of(), true));
     }
 
     private static Layout layout(FieldAttribute... fields) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/VerifierChecks.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/VerifierChecks.java
index 44ac786ba32fe..36ce187d8600c 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/VerifierChecks.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/VerifierChecks.java
@@ -15,7 +15,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.xpack.esql.core.common.Failure.fail;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
 
 public final class VerifierChecks {
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/AbstractFieldHitExtractor.java
index 6a8bd61f89c03..9f7155a78e66f 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/AbstractFieldHitExtractor.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/AbstractFieldHitExtractor.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.io.IOException;
 import java.time.ZoneId;
@@ -85,7 +84,7 @@ protected AbstractFieldHitExtractor(StreamInput in) throws IOException {
     }
 
     protected DataType loadTypeFromName(String typeName) {
-        return DataTypes.fromTypeName(typeName);
+        return DataType.fromTypeName(typeName);
     }
 
     protected abstract ZoneId readZoneId(StreamInput in) throws IOException;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java
index 7a724eaa2be65..5824358e57525 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
 import java.io.IOException;
@@ -60,7 +59,7 @@ public boolean resolved() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.NULL;
+        return DataType.NULL;
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java
index 980a0312c29f0..8baffbf887e47 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.esql.core.expression;
 
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -133,7 +133,7 @@ public static String name(Expression e) {
     }
 
     public static boolean isNull(Expression e) {
-        return e.dataType() == DataTypes.NULL || (e.foldable() && e.fold() == null);
+        return e.dataType() == DataType.NULL || (e.foldable() && e.fold() == null);
     }
 
     public static List<String> names(Collection<? extends Expression> e) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java
index ea37629df6085..68780f5b32e9c 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.util.Objects;
 
@@ -19,9 +18,9 @@
  */
 public class Literal extends LeafExpression {
 
-    public static final Literal TRUE = new Literal(Source.EMPTY, Boolean.TRUE, DataTypes.BOOLEAN);
-    public static final Literal FALSE = new Literal(Source.EMPTY, Boolean.FALSE, DataTypes.BOOLEAN);
-    public static final Literal NULL = new Literal(Source.EMPTY, null, DataTypes.NULL);
+    public static final Literal TRUE = new Literal(Source.EMPTY, Boolean.TRUE, DataType.BOOLEAN);
+    public static final Literal FALSE = new Literal(Source.EMPTY, Boolean.FALSE, DataType.BOOLEAN);
+    public static final Literal NULL = new Literal(Source.EMPTY, null, DataType.NULL);
 
     private final Object value;
     private final DataType dataType;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
index 9cbee26f443ba..ed9ad996137e6 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
 
 import java.io.IOException;
@@ -35,15 +34,15 @@ public class MetadataAttribute extends TypedAttribute {
 
     private static final Map<String, Tuple<DataType, Boolean>> ATTRIBUTES_MAP = Map.of(
         "_version",
-        tuple(DataTypes.LONG, false), // _version field is not searchable
+        tuple(DataType.LONG, false), // _version field is not searchable
         "_index",
-        tuple(DataTypes.KEYWORD, true),
+        tuple(DataType.KEYWORD, true),
         IdFieldMapper.NAME,
-        tuple(DataTypes.KEYWORD, false), // actually searchable, but fielddata access on the _id field is disallowed by default
+        tuple(DataType.KEYWORD, false), // actually searchable, but fielddata access on the _id field is disallowed by default
         IgnoredFieldMapper.NAME,
-        tuple(DataTypes.KEYWORD, true),
+        tuple(DataType.KEYWORD, true),
         SourceFieldMapper.NAME,
-        tuple(DataTypes.SOURCE, false)
+        tuple(DataType.SOURCE, false)
     );
 
     private final boolean searchable;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
index a0b61022c1450..588b0a2af55d3 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java
@@ -8,7 +8,6 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression.TypeResolution;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 
 import java.util.Locale;
@@ -18,10 +17,10 @@
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.core.expression.Expressions.name;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NULL;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
 
 public final class TypeResolutions {
 
@@ -60,7 +59,7 @@ public static TypeResolution isNumeric(Expression e, String operationName, Param
     }
 
     public static TypeResolution isString(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return isType(e, DataTypes::isString, operationName, paramOrd, "string");
+        return isType(e, DataType::isString, operationName, paramOrd, "string");
     }
 
     public static TypeResolution isIP(Expression e, String operationName, ParamOrdinal paramOrd) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/string/BinaryComparisonCaseInsensitiveFunction.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/string/BinaryComparisonCaseInsensitiveFunction.java
index 920e77d52d0d3..4739fe910b769 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/string/BinaryComparisonCaseInsensitiveFunction.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/string/BinaryComparisonCaseInsensitiveFunction.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.util.Objects;
 
@@ -53,7 +52,7 @@ public Expression right() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/whitelist/InternalQlScriptUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/whitelist/InternalQlScriptUtils.java
index 8895a2dc5473c..e361d2465a1c5 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/whitelist/InternalQlScriptUtils.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/whitelist/InternalQlScriptUtils.java
@@ -22,9 +22,9 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.core.type.DataType.fromTypeName;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.convert;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.toUnsignedLong;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.fromTypeName;
 
 public class InternalQlScriptUtils {
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java
index 9d0dd9b161b4d..ee48fd84b8add 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.DateUtils;
 
 import java.time.DateTimeException;
@@ -104,7 +103,7 @@ public Object fold() {
     protected boolean areBoundariesInvalid() {
         Object lowerValue = lower.fold();
         Object upperValue = upper.fold();
-        if (DataTypes.isDateTime(value.dataType()) || DataTypes.isDateTime(lower.dataType()) || DataTypes.isDateTime(upper.dataType())) {
+        if (DataType.isDateTime(value.dataType()) || DataType.isDateTime(lower.dataType()) || DataType.isDateTime(upper.dataType())) {
             try {
                 if (upperValue instanceof String upperString) {
                     upperValue = DateUtils.asDateTime(upperString);
@@ -127,7 +126,7 @@ protected boolean areBoundariesInvalid() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java
index 06b70b4afa27c..8da858865ed3f 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.util.List;
 import java.util.Map;
@@ -65,7 +64,7 @@ public Nullability nullable() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/BinaryLogic.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/BinaryLogic.java
index a7145a7acfe29..39de0e0643c13 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/BinaryLogic.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/BinaryLogic.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isBoolean;
 
@@ -25,7 +24,7 @@ protected BinaryLogic(Source source, Expression left, Expression right, BinaryLo
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java
index 379e67a3e599d..31c63393afaea 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isBoolean;
@@ -36,7 +35,7 @@ protected Not replaceChild(Expression newChild) {
 
     @Override
     protected TypeResolution resolveType() {
-        if (DataTypes.BOOLEAN == field().dataType()) {
+        if (DataType.BOOLEAN == field().dataType()) {
             return TypeResolution.TYPE_RESOLVED;
         }
         return isBoolean(field(), sourceText(), DEFAULT);
@@ -67,7 +66,7 @@ public Expression negate() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     static Expression negate(Expression exp) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java
index 5eebbd1294cc5..52375c5db01a1 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 public class IsNotNull extends UnaryScalarFunction implements Negatable<UnaryScalarFunction> {
 
@@ -35,7 +34,7 @@ protected IsNotNull replaceChild(Expression newChild) {
 
     @Override
     public Object fold() {
-        return field().fold() != null && DataTypes.isNull(field().dataType()) == false;
+        return field().fold() != null && DataType.isNull(field().dataType()) == false;
     }
 
     @Override
@@ -50,7 +49,7 @@ public Nullability nullable() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java
index 56c8fa578da39..d52eec9114df6 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 public class IsNull extends UnaryScalarFunction implements Negatable<UnaryScalarFunction> {
 
@@ -35,7 +34,7 @@ protected IsNull replaceChild(Expression newChild) {
 
     @Override
     public Object fold() {
-        return field().fold() == null || DataTypes.isNull(field().dataType());
+        return field().fold() == null || DataType.isNull(field().dataType());
     }
 
     @Override
@@ -50,7 +49,7 @@ public Nullability nullable() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mul.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mul.java
index 46cc72766d531..db46ecf81ea1d 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mul.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mul.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 
@@ -33,7 +32,7 @@ protected TypeResolution resolveType() {
         DataType r = right().dataType();
 
         // 1. both are numbers
-        if (DataTypes.isNullOrNumeric(l) && DataTypes.isNullOrNumeric(r)) {
+        if (DataType.isNullOrNumeric(l) && DataType.isNullOrNumeric(r)) {
             return TypeResolution.TYPE_RESOLVED;
         }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/BinaryComparison.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/BinaryComparison.java
index a9c4fd439a1ff..193b77f2344c0 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/BinaryComparison.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/BinaryComparison.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.time.ZoneId;
 
@@ -38,7 +37,7 @@ protected TypeResolution resolveInputType(Expression e, ParamOrdinal paramOrdina
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     public static Integer compare(Object left, Object right) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/In.java
index ff188cb30d7d6..21fbfa56b0d98 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/In.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/comparison/In.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypeConverter;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 
 import java.time.ZoneId;
@@ -71,7 +70,7 @@ public List<Expression> list() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
@@ -110,7 +109,7 @@ protected List<Object> foldAndConvertListOfValues(List<Expression> expressions,
     }
 
     protected boolean areCompatible(DataType left, DataType right) {
-        return DataTypes.areCompatible(left, right);
+        return DataType.areCompatible(left, right);
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java
index 084bdb06ca6d2..4e7e70685dc3a 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.util.Objects;
 
@@ -42,7 +41,7 @@ public boolean caseInsensitive() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexCompatibility.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexCompatibility.java
index e0ee089680254..6cc0816661f01 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexCompatibility.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexCompatibility.java
@@ -15,7 +15,7 @@
 import java.util.Map;
 
 import static org.elasticsearch.xpack.esql.core.index.VersionCompatibilityChecks.isTypeSupportedInVersion;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isPrimitive;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isPrimitive;
 import static org.elasticsearch.xpack.esql.core.type.Types.propagateUnsupportedType;
 
 public final class IndexCompatibility {
@@ -27,9 +27,9 @@ public static Map<String, EsField> compatible(Map<String, EsField> mapping, Vers
             if (isPrimitive(dataType) == false) {
                 compatible(esField.getProperties(), version);
             } else if (isTypeSupportedInVersion(dataType, version) == false) {
-                EsField field = new UnsupportedEsField(entry.getKey(), dataType.name(), null, esField.getProperties());
+                EsField field = new UnsupportedEsField(entry.getKey(), dataType.nameUpper(), null, esField.getProperties());
                 entry.setValue(field);
-                propagateUnsupportedType(entry.getKey(), dataType.name(), esField.getProperties());
+                propagateUnsupportedType(entry.getKey(), dataType.nameUpper(), esField.getProperties());
             }
         }
         return mapping;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexResolver.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexResolver.java
index 21222b5d36a55..63467eaadd8df 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexResolver.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexResolver.java
@@ -65,11 +65,11 @@
 import static org.elasticsearch.common.Strings.hasText;
 import static org.elasticsearch.common.regex.Regex.simpleMatch;
 import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.OBJECT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSUPPORTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.qualifyAndJoinIndices;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.splitQualifiedIndex;
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/VersionCompatibilityChecks.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/VersionCompatibilityChecks.java
index ce6d26bc3528f..e4ae4f8f0d51f 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/VersionCompatibilityChecks.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/VersionCompatibilityChecks.java
@@ -15,8 +15,8 @@
 
 import static org.elasticsearch.Version.V_8_2_0;
 import static org.elasticsearch.Version.V_8_4_0;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 
 public final class VersionCompatibilityChecks {
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java
index 12b496e51fa1b..7759b62d3d187 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java
@@ -45,7 +45,7 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.rule.Rule;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.ReflectionUtils;
 
@@ -143,7 +143,7 @@ private static Expression simplifyAndOr(BinaryPredicate<?, ?, ?, ?> bc) {
                 }
 
                 if (FALSE.equals(l) || FALSE.equals(r)) {
-                    return new Literal(bc.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                    return new Literal(bc.source(), Boolean.FALSE, DataType.BOOLEAN);
                 }
                 if (l.semanticEquals(r)) {
                     return l;
@@ -173,7 +173,7 @@ private static Expression simplifyAndOr(BinaryPredicate<?, ?, ?, ?> bc) {
 
             if (bc instanceof Or) {
                 if (TRUE.equals(l) || TRUE.equals(r)) {
-                    return new Literal(bc.source(), Boolean.TRUE, DataTypes.BOOLEAN);
+                    return new Literal(bc.source(), Boolean.TRUE, DataType.BOOLEAN);
                 }
 
                 if (FALSE.equals(l)) {
@@ -218,10 +218,10 @@ private Expression simplifyNot(Not n) {
             Expression c = n.field();
 
             if (TRUE.semanticEquals(c)) {
-                return new Literal(n.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                return new Literal(n.source(), Boolean.FALSE, DataType.BOOLEAN);
             }
             if (FALSE.semanticEquals(c)) {
-                return new Literal(n.source(), Boolean.TRUE, DataTypes.BOOLEAN);
+                return new Literal(n.source(), Boolean.TRUE, DataType.BOOLEAN);
             }
 
             Expression negated = maybeSimplifyNegatable(c);
@@ -262,12 +262,12 @@ protected Expression rule(BinaryComparison bc) {
             // true for equality
             if (bc instanceof Equals || bc instanceof GreaterThanOrEqual || bc instanceof LessThanOrEqual) {
                 if (l.nullable() == Nullability.FALSE && r.nullable() == Nullability.FALSE && l.semanticEquals(r)) {
-                    return new Literal(bc.source(), Boolean.TRUE, DataTypes.BOOLEAN);
+                    return new Literal(bc.source(), Boolean.TRUE, DataType.BOOLEAN);
                 }
             }
             if (bc instanceof NullEquals) {
                 if (l.semanticEquals(r)) {
-                    return new Literal(bc.source(), Boolean.TRUE, DataTypes.BOOLEAN);
+                    return new Literal(bc.source(), Boolean.TRUE, DataType.BOOLEAN);
                 }
                 if (Expressions.isNull(r)) {
                     return new IsNull(bc.source(), l);
@@ -277,7 +277,7 @@ protected Expression rule(BinaryComparison bc) {
             // false for equality
             if (bc instanceof NotEquals || bc instanceof GreaterThan || bc instanceof LessThan) {
                 if (l.nullable() == Nullability.FALSE && r.nullable() == Nullability.FALSE && l.semanticEquals(r)) {
-                    return new Literal(bc.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                    return new Literal(bc.source(), Boolean.FALSE, DataType.BOOLEAN);
                 }
             }
 
@@ -343,14 +343,14 @@ private static Expression propagate(And and) {
                     BinaryComparison otherEq = (BinaryComparison) ex;
                     // equals on different values evaluate to FALSE
                     // ignore date/time fields as equality comparison might actually be a range check
-                    if (otherEq.right().foldable() && DataTypes.isDateTime(otherEq.left().dataType()) == false) {
+                    if (otherEq.right().foldable() && DataType.isDateTime(otherEq.left().dataType()) == false) {
                         for (BinaryComparison eq : equals) {
                             if (otherEq.left().semanticEquals(eq.left())) {
                                 Integer comp = BinaryComparison.compare(eq.right().fold(), otherEq.right().fold());
                                 if (comp != null) {
                                     // var cannot be equal to two different values at the same time
                                     if (comp != 0) {
-                                        return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                        return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                                     }
                                 }
                             }
@@ -396,7 +396,7 @@ private static Expression propagate(And and) {
                             compare > 0 ||
                             // eq matches the boundary but should not be included
                                 (compare == 0 && range.includeLower() == false))) {
-                                return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                             }
                         }
                         if (range.upper().foldable()) {
@@ -406,7 +406,7 @@ private static Expression propagate(And and) {
                             compare < 0 ||
                             // eq matches the boundary but should not be included
                                 (compare == 0 && range.includeUpper() == false))) {
-                                return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                             }
                         }
 
@@ -423,7 +423,7 @@ private static Expression propagate(And and) {
                         Integer comp = BinaryComparison.compare(eqValue, neq.right().fold());
                         if (comp != null) {
                             if (comp == 0) { // clashing and conflicting: a = 1 AND a != 1
-                                return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                             } else { // clashing and redundant: a = 1 AND a != 2
                                 iter.remove();
                                 changed = true;
@@ -441,12 +441,12 @@ private static Expression propagate(And and) {
                             if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { // a = 2 AND a </<= ?
                                 if ((compare == 0 && bc instanceof LessThan) || // a = 2 AND a < 2
                                     0 < compare) { // a = 2 AND a </<= 1
-                                    return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                    return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                                 }
                             } else if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) { // a = 2 AND a >/>= ?
                                 if ((compare == 0 && bc instanceof GreaterThan) || // a = 2 AND a > 2
                                     compare < 0) { // a = 2 AND a >/>= 3
-                                    return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                    return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                                 }
                             }
 
@@ -1296,7 +1296,7 @@ private static Expression foldBinaryLogic(BinaryLogic binaryLogic) {
                 boolean nullLeft = Expressions.isNull(or.left());
                 boolean nullRight = Expressions.isNull(or.right());
                 if (nullLeft && nullRight) {
-                    return new Literal(binaryLogic.source(), null, DataTypes.NULL);
+                    return new Literal(binaryLogic.source(), null, DataType.NULL);
                 }
                 if (nullLeft) {
                     return or.right();
@@ -1307,7 +1307,7 @@ private static Expression foldBinaryLogic(BinaryLogic binaryLogic) {
             }
             if (binaryLogic instanceof And and) {
                 if (Expressions.isNull(and.left()) || Expressions.isNull(and.right())) {
-                    return new Literal(binaryLogic.source(), null, DataTypes.NULL);
+                    return new Literal(binaryLogic.source(), null, DataType.NULL);
                 }
             }
             return binaryLogic;
@@ -1391,7 +1391,7 @@ protected Expression rule(RegexMatch<?> regexMatch) {
             } else {
                 String match = pattern.exactMatch();
                 if (match != null) {
-                    Literal literal = new Literal(regexMatch.source(), match, DataTypes.KEYWORD);
+                    Literal literal = new Literal(regexMatch.source(), match, DataType.KEYWORD);
                     e = regexToEquals(regexMatch, literal);
                 }
             }
@@ -1429,11 +1429,11 @@ protected Expression rule(Expression e) {
         protected Expression tryReplaceIsNullIsNotNull(Expression e) {
             if (e instanceof IsNotNull isnn) {
                 if (isnn.field().nullable() == Nullability.FALSE) {
-                    return new Literal(e.source(), Boolean.TRUE, DataTypes.BOOLEAN);
+                    return new Literal(e.source(), Boolean.TRUE, DataType.BOOLEAN);
                 }
             } else if (e instanceof IsNull isn) {
                 if (isn.field().nullable() == Nullability.FALSE) {
-                    return new Literal(e.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                    return new Literal(e.source(), Boolean.FALSE, DataType.BOOLEAN);
                 }
             }
             return e;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
index cf19b8722b258..6eab4a0cd9a75 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java
@@ -52,7 +52,6 @@
 import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.Check;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.versionfield.Version;
@@ -67,9 +66,9 @@
 import java.util.List;
 import java.util.Set;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
 
 public final class ExpressionTranslators {
@@ -283,7 +282,7 @@ static Query translate(BinaryComparison bc, TranslatorHandler handler) {
             }
 
             ZoneId zoneId = null;
-            if (DataTypes.isDateTime(attribute.dataType())) {
+            if (DataType.isDateTime(attribute.dataType())) {
                 zoneId = bc.zoneId();
             }
             if (bc instanceof GreaterThan) {
@@ -379,7 +378,7 @@ public static Query doTranslate(In in, TranslatorHandler handler) {
         }
 
         private static boolean needsTypeSpecificValueHandling(DataType fieldType) {
-            return DataTypes.isDateTime(fieldType) || fieldType == IP || fieldType == VERSION || fieldType == UNSIGNED_LONG;
+            return DataType.isDateTime(fieldType) || fieldType == IP || fieldType == VERSION || fieldType == UNSIGNED_LONG;
         }
 
         private static Query translate(In in, TranslatorHandler handler) {
@@ -389,7 +388,7 @@ private static Query translate(In in, TranslatorHandler handler) {
             List<Query> queries = new ArrayList<>();
 
             for (Expression rhs : in.list()) {
-                if (DataTypes.isNull(rhs.dataType()) == false) {
+                if (DataType.isNull(rhs.dataType()) == false) {
                     if (needsTypeSpecificValueHandling(attribute.dataType())) {
                         // delegates to BinaryComparisons translator to ensure consistent handling of date and time values
                         Query query = BinaryComparisons.translate(new Equals(in.source(), in.value(), rhs, in.zoneId()), handler);
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
index 1dacc394c8d21..9d6a325a6028f 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
@@ -4,19 +4,65 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
 package org.elasticsearch.xpack.esql.core.type;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.search.sort.ScriptSortBuilder;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 
 import java.io.IOException;
+import java.math.BigInteger;
+import java.time.ZonedDateTime;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.Locale;
-import java.util.Objects;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toUnmodifiableMap;
+
+public enum DataType {
+    UNSUPPORTED("UNSUPPORTED", null, 0, false, false, false),
+    NULL("null", 0, false, false, false),
+    BOOLEAN("boolean", 1, false, false, false),
+    BYTE("byte", Byte.BYTES, true, false, true),
+    SHORT("short", Short.BYTES, true, false, true),
+    INTEGER("integer", Integer.BYTES, true, false, true),
+    LONG("long", Long.BYTES, true, false, true),
+    UNSIGNED_LONG("unsigned_long", Long.BYTES, true, false, true),
+    DOUBLE("double", Double.BYTES, false, true, true),
+    FLOAT("float", Float.BYTES, false, true, true),
+    HALF_FLOAT("half_float", Float.BYTES, false, true, true),
+    SCALED_FLOAT("scaled_float", Long.BYTES, false, true, true),
+    KEYWORD("keyword", Integer.MAX_VALUE, false, false, true),
+    TEXT("text", Integer.MAX_VALUE, false, false, false),
+    DATETIME("DATETIME", "date", Long.BYTES, false, false, true),
+    IP("ip", 45, false, false, true),
+    VERSION("version", Integer.MAX_VALUE, false, false, true),
+    OBJECT("object", 0, false, false, false),
+    NESTED("nested", 0, false, false, false),
+    SOURCE(SourceFieldMapper.NAME, SourceFieldMapper.NAME, Integer.MAX_VALUE, false, false, false),
+    DATE_PERIOD("DATE_PERIOD", null, 3 * Integer.BYTES, false, false, false),
+    TIME_DURATION("TIME_DURATION", null, Integer.BYTES + Long.BYTES, false, false, false),
+    GEO_POINT("geo_point", Double.BYTES * 2, false, false, true),
+    CARTESIAN_POINT("cartesian_point", Double.BYTES * 2, false, false, true),
+    CARTESIAN_SHAPE("cartesian_shape", Integer.MAX_VALUE, false, false, true),
+    GEO_SHAPE("geo_shape", Integer.MAX_VALUE, false, false, true),
 
-public class DataType implements Writeable {
+    /**
+     * These are numeric fields labeled as metric counters in time-series indices. Although stored
+     * internally as numeric fields, they represent cumulative metrics and must not be treated as regular
+     * numeric fields. Therefore, we define them differently and separately from their parent numeric field.
+     * These fields are strictly for use in retrieval from indices, rate aggregation, and casting to their
+     * parent numeric type.
+     */
+    COUNTER_LONG("counter_long", Long.BYTES, false, false, true),
+    COUNTER_INTEGER("counter_integer", Integer.BYTES, false, false, true),
+    COUNTER_DOUBLE("counter_double", Double.BYTES, false, false, true),
+    DOC_DATA_TYPE("_doc", Integer.BYTES * 3, false, false, false),
+    TSID_DATA_TYPE("_tsid", Integer.MAX_VALUE, false, false, true);
 
     private final String typeName;
 
@@ -41,11 +87,11 @@ public class DataType implements Writeable {
      */
     private final boolean docValues;
 
-    public DataType(String esName, int size, boolean isInteger, boolean isRational, boolean hasDocValues) {
+    DataType(String esName, int size, boolean isInteger, boolean isRational, boolean hasDocValues) {
         this(null, esName, size, isInteger, isRational, hasDocValues);
     }
 
-    public DataType(String typeName, String esType, int size, boolean isInteger, boolean isRational, boolean hasDocValues) {
+    DataType(String typeName, String esType, int size, boolean isInteger, boolean isRational, boolean hasDocValues) {
         String typeString = typeName != null ? typeName : esType;
         this.typeName = typeString.toLowerCase(Locale.ROOT);
         this.name = typeString.toUpperCase(Locale.ROOT);
@@ -56,7 +102,139 @@ public DataType(String typeName, String esType, int size, boolean isInteger, boo
         this.docValues = hasDocValues;
     }
 
-    public String name() {
+    private static final Collection<DataType> TYPES = Stream.of(
+        UNSUPPORTED,
+        NULL,
+        BOOLEAN,
+        BYTE,
+        SHORT,
+        INTEGER,
+        LONG,
+        UNSIGNED_LONG,
+        DOUBLE,
+        FLOAT,
+        HALF_FLOAT,
+        SCALED_FLOAT,
+        KEYWORD,
+        TEXT,
+        DATETIME,
+        IP,
+        VERSION,
+        OBJECT,
+        NESTED,
+        SOURCE,
+        DATE_PERIOD,
+        TIME_DURATION,
+        GEO_POINT,
+        CARTESIAN_POINT,
+        CARTESIAN_SHAPE,
+        GEO_SHAPE,
+        COUNTER_LONG,
+        COUNTER_INTEGER,
+        COUNTER_DOUBLE
+    ).sorted(Comparator.comparing(DataType::typeName)).toList();
+
+    private static final Map<String, DataType> NAME_TO_TYPE = TYPES.stream().collect(toUnmodifiableMap(DataType::typeName, t -> t));
+
+    private static Map<String, DataType> ES_TO_TYPE;
+
+    static {
+        Map<String, DataType> map = TYPES.stream().filter(e -> e.esType() != null).collect(toMap(DataType::esType, t -> t));
+        map.put("date_nanos", DATETIME);
+        ES_TO_TYPE = Collections.unmodifiableMap(map);
+    }
+
+    public static Collection<DataType> types() {
+        return TYPES;
+    }
+
+    public static DataType fromTypeName(String name) {
+        return NAME_TO_TYPE.get(name.toLowerCase(Locale.ROOT));
+    }
+
+    public static DataType fromEs(String name) {
+        DataType type = ES_TO_TYPE.get(name);
+        return type != null ? type : UNSUPPORTED;
+    }
+
+    public static DataType fromJava(Object value) {
+        if (value == null) {
+            return NULL;
+        }
+        if (value instanceof Integer) {
+            return INTEGER;
+        }
+        if (value instanceof Long) {
+            return LONG;
+        }
+        if (value instanceof BigInteger) {
+            return UNSIGNED_LONG;
+        }
+        if (value instanceof Boolean) {
+            return BOOLEAN;
+        }
+        if (value instanceof Double) {
+            return DOUBLE;
+        }
+        if (value instanceof Float) {
+            return FLOAT;
+        }
+        if (value instanceof Byte) {
+            return BYTE;
+        }
+        if (value instanceof Short) {
+            return SHORT;
+        }
+        if (value instanceof ZonedDateTime) {
+            return DATETIME;
+        }
+        if (value instanceof String || value instanceof Character) {
+            return KEYWORD;
+        }
+
+        return null;
+    }
+
+    public static boolean isUnsupported(DataType from) {
+        return from == UNSUPPORTED;
+    }
+
+    public static boolean isString(DataType t) {
+        return t == KEYWORD || t == TEXT;
+    }
+
+    public static boolean isPrimitive(DataType t) {
+        return t != OBJECT && t != NESTED && t != UNSUPPORTED;
+    }
+
+    public static boolean isNull(DataType t) {
+        return t == NULL;
+    }
+
+    public static boolean isNullOrNumeric(DataType t) {
+        return t.isNumeric() || isNull(t);
+    }
+
+    public static boolean isSigned(DataType t) {
+        return t.isNumeric() && t.equals(UNSIGNED_LONG) == false;
+    }
+
+    public static boolean isDateTime(DataType type) {
+        return type == DATETIME;
+    }
+
+    public static boolean areCompatible(DataType left, DataType right) {
+        if (left == right) {
+            return true;
+        } else {
+            return (left == NULL || right == NULL)
+                || (isString(left) && isString(right))
+                || (left.isNumeric() && right.isNumeric())
+                || (isDateTime(left) && isDateTime(right));
+        }
+    }
+
+    public String nameUpper() {
         return name;
     }
 
@@ -68,12 +246,6 @@ public String esType() {
         return esType;
     }
 
-    public ScriptSortBuilder.ScriptSortType scriptSortType() {
-        return isNumeric() ? ScriptSortBuilder.ScriptSortType.NUMBER
-            : this == DataTypes.VERSION ? ScriptSortBuilder.ScriptSortType.VERSION
-            : ScriptSortBuilder.ScriptSortType.STRING;
-    }
-
     public boolean isInteger() {
         return isInteger;
     }
@@ -94,49 +266,20 @@ public boolean hasDocValues() {
         return docValues;
     }
 
-    @Override
-    public int hashCode() {
-        return Objects.hash(typeName, esType, size, isInteger, isRational, docValues);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-
-        if (obj == null || getClass() != obj.getClass()) {
-            return false;
-        }
-
-        DataType other = (DataType) obj;
-        return Objects.equals(typeName, other.typeName)
-            && Objects.equals(esType, other.esType)
-            && size == other.size
-            && isInteger == other.isInteger
-            && isRational == other.isRational
-            && docValues == other.docValues;
-    }
-
-    @Override
-    public String toString() {
-        return name;
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(typeName);
     }
 
     public static DataType readFrom(StreamInput in) throws IOException {
+        // TODO: Use our normal enum serialization pattern
         String name = in.readString();
-        if (name.equalsIgnoreCase(DataTypes.DOC_DATA_TYPE.name())) {
-            return DataTypes.DOC_DATA_TYPE;
+        if (name.equalsIgnoreCase(DataType.DOC_DATA_TYPE.nameUpper())) {
+            return DataType.DOC_DATA_TYPE;
         }
-        DataType dataType = DataTypes.fromTypeName(name);
+        DataType dataType = DataType.fromTypeName(name);
         if (dataType == null) {
             throw new IOException("Unknown DataType for type name: " + name);
         }
         return dataType;
     }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(typeName);
-    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java
index 03466655ccc32..bb53472d06e71 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypeConverter.java
@@ -22,23 +22,23 @@
 import java.util.function.DoubleFunction;
 import java.util.function.Function;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BYTE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NULL;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.SHORT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isDateTime;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isPrimitive;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isString;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isPrimitive;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isString;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.UNSIGNED_LONG_MAX;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.inUnsignedLongRange;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.isUnsignedLong;
@@ -470,7 +470,7 @@ public static boolean convertToBoolean(String val) {
      * Throws InvalidArgumentException if such conversion is not possible
      */
     public static Object convert(Object value, DataType dataType) {
-        DataType detectedType = DataTypes.fromJava(value);
+        DataType detectedType = DataType.fromJava(value);
         if (detectedType == dataType || value == null) {
             return value;
         }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java
deleted file mode 100644
index 8d4129d1abbcc..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataTypes.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.type;
-
-import org.elasticsearch.index.mapper.SourceFieldMapper;
-
-import java.math.BigInteger;
-import java.time.ZonedDateTime;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Locale;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import static java.util.stream.Collectors.toMap;
-import static java.util.stream.Collectors.toUnmodifiableMap;
-
-public final class DataTypes {
-
-    // tag::noformat
-    public static final DataType UNSUPPORTED      = new DataType("UNSUPPORTED", null, 0,                 false, false, false);
-
-    public static final DataType NULL             = new DataType("null",              0,                 false, false, false);
-
-    public static final DataType BOOLEAN          = new DataType("boolean",           1,                 false, false, false);
-    // integer numeric
-    public static final DataType BYTE             = new DataType("byte",              Byte.BYTES,        true, false, true);
-    public static final DataType SHORT            = new DataType("short",             Short.BYTES,       true, false, true);
-    public static final DataType INTEGER          = new DataType("integer",           Integer.BYTES,     true, false, true);
-    public static final DataType LONG             = new DataType("long",              Long.BYTES,        true, false, true);
-    public static final DataType UNSIGNED_LONG    = new DataType("unsigned_long",     Long.BYTES,        true, false, true);
-    // decimal numeric
-    public static final DataType DOUBLE           = new DataType("double",            Double.BYTES,      false, true, true);
-    public static final DataType FLOAT            = new DataType("float",             Float.BYTES,       false, true, true);
-    public static final DataType HALF_FLOAT       = new DataType("half_float",        Float.BYTES,       false, true, true);
-    public static final DataType SCALED_FLOAT     = new DataType("scaled_float",      Long.BYTES,        false, true, true);
-    // string
-    public static final DataType KEYWORD          = new DataType("keyword",           Integer.MAX_VALUE, false, false, true);
-    public static final DataType TEXT             = new DataType("text",              Integer.MAX_VALUE, false, false, false);
-    // date
-    public static final DataType DATETIME         = new DataType("DATETIME", "date",        Long.BYTES,  false, false, true);
-    // ip
-    public static final DataType IP               = new DataType("ip",                45,                false, false, true);
-    // version
-    public static final DataType VERSION          = new DataType("version",           Integer.MAX_VALUE, false, false, true);
-    // complex types
-    public static final DataType OBJECT           = new DataType("object",            0,                 false, false, false);
-    public static final DataType NESTED           = new DataType("nested",            0,                 false, false, false);
-    //end::noformat
-    public static final DataType SOURCE = new DataType(
-        SourceFieldMapper.NAME,
-        SourceFieldMapper.NAME,
-        Integer.MAX_VALUE,
-        false,
-        false,
-        false
-    );
-    public static final DataType DATE_PERIOD = new DataType("DATE_PERIOD", null, 3 * Integer.BYTES, false, false, false);
-    public static final DataType TIME_DURATION = new DataType("TIME_DURATION", null, Integer.BYTES + Long.BYTES, false, false, false);
-    public static final DataType GEO_POINT = new DataType("geo_point", Double.BYTES * 2, false, false, true);
-    public static final DataType CARTESIAN_POINT = new DataType("cartesian_point", Double.BYTES * 2, false, false, true);
-    public static final DataType GEO_SHAPE = new DataType("geo_shape", Integer.MAX_VALUE, false, false, true);
-    public static final DataType CARTESIAN_SHAPE = new DataType("cartesian_shape", Integer.MAX_VALUE, false, false, true);
-
-    /**
-     * These are numeric fields labeled as metric counters in time-series indices. Although stored
-     * internally as numeric fields, they represent cumulative metrics and must not be treated as regular
-     * numeric fields. Therefore, we define them differently and separately from their parent numeric field.
-     * These fields are strictly for use in retrieval from indices, rate aggregation, and casting to their
-     * parent numeric type.
-     */
-    public static final DataType COUNTER_LONG = new DataType("counter_long", Long.BYTES, false, false, true);
-    public static final DataType COUNTER_INTEGER = new DataType("counter_integer", Integer.BYTES, false, false, true);
-    public static final DataType COUNTER_DOUBLE = new DataType("counter_double", Double.BYTES, false, false, true);
-
-    public static final DataType DOC_DATA_TYPE = new DataType("_doc", Integer.BYTES * 3, false, false, false);
-    public static final DataType TSID_DATA_TYPE = new DataType("_tsid", Integer.MAX_VALUE, false, false, true);
-
-    private static final Collection<DataType> TYPES = Stream.of(
-        UNSUPPORTED,
-        NULL,
-        BOOLEAN,
-        BYTE,
-        SHORT,
-        INTEGER,
-        LONG,
-        UNSIGNED_LONG,
-        DOUBLE,
-        FLOAT,
-        HALF_FLOAT,
-        SCALED_FLOAT,
-        KEYWORD,
-        TEXT,
-        DATETIME,
-        IP,
-        VERSION,
-        OBJECT,
-        NESTED,
-        SOURCE,
-        DATE_PERIOD,
-        TIME_DURATION,
-        GEO_POINT,
-        CARTESIAN_POINT,
-        CARTESIAN_SHAPE,
-        GEO_SHAPE,
-        COUNTER_LONG,
-        COUNTER_INTEGER,
-        COUNTER_DOUBLE
-    ).sorted(Comparator.comparing(DataType::typeName)).toList();
-
-    private static final Map<String, DataType> NAME_TO_TYPE = TYPES.stream().collect(toUnmodifiableMap(DataType::typeName, t -> t));
-
-    private static Map<String, DataType> ES_TO_TYPE;
-
-    static {
-        Map<String, DataType> map = TYPES.stream().filter(e -> e.esType() != null).collect(toMap(DataType::esType, t -> t));
-        map.put("date_nanos", DATETIME);
-        ES_TO_TYPE = Collections.unmodifiableMap(map);
-    }
-
-    private DataTypes() {}
-
-    public static Collection<DataType> types() {
-        return TYPES;
-    }
-
-    public static DataType fromTypeName(String name) {
-        return NAME_TO_TYPE.get(name.toLowerCase(Locale.ROOT));
-    }
-
-    public static DataType fromEs(String name) {
-        DataType type = ES_TO_TYPE.get(name);
-        return type != null ? type : UNSUPPORTED;
-    }
-
-    public static DataType fromJava(Object value) {
-        if (value == null) {
-            return NULL;
-        }
-        if (value instanceof Integer) {
-            return INTEGER;
-        }
-        if (value instanceof Long) {
-            return LONG;
-        }
-        if (value instanceof BigInteger) {
-            return UNSIGNED_LONG;
-        }
-        if (value instanceof Boolean) {
-            return BOOLEAN;
-        }
-        if (value instanceof Double) {
-            return DOUBLE;
-        }
-        if (value instanceof Float) {
-            return FLOAT;
-        }
-        if (value instanceof Byte) {
-            return BYTE;
-        }
-        if (value instanceof Short) {
-            return SHORT;
-        }
-        if (value instanceof ZonedDateTime) {
-            return DATETIME;
-        }
-        if (value instanceof String || value instanceof Character) {
-            return KEYWORD;
-        }
-
-        return null;
-    }
-
-    public static boolean isUnsupported(DataType from) {
-        return from == UNSUPPORTED;
-    }
-
-    public static boolean isString(DataType t) {
-        return t == KEYWORD || t == TEXT;
-    }
-
-    public static boolean isPrimitive(DataType t) {
-        return t != OBJECT && t != NESTED && t != UNSUPPORTED;
-    }
-
-    public static boolean isNull(DataType t) {
-        return t == NULL;
-    }
-
-    public static boolean isNullOrNumeric(DataType t) {
-        return t.isNumeric() || isNull(t);
-    }
-
-    public static boolean isSigned(DataType t) {
-        return t.isNumeric() && t.equals(UNSIGNED_LONG) == false;
-    }
-
-    public static boolean isDateTime(DataType type) {
-        return type == DATETIME;
-    }
-
-    public static boolean areCompatible(DataType left, DataType right) {
-        if (left == right) {
-            return true;
-        } else {
-            return (left == NULL || right == NULL)
-                || (isString(left) && isString(right))
-                || (left.isNumeric() && right.isNumeric())
-                || (isDateTime(left) && isDateTime(right));
-        }
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
index 9d2341f77505e..01728954a2e1b 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java
@@ -20,7 +20,7 @@ public class DateEsField extends EsField {
     static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(EsField.class, "DateEsField", DateEsField::new);
 
     public static DateEsField dateEsField(String name, Map<String, EsField> properties, boolean hasDocValues) {
-        return new DateEsField(name, DataTypes.DATETIME, properties, hasDocValues);
+        return new DateEsField(name, DataType.DATETIME, properties, hasDocValues);
     }
 
     private DateEsField(String name, DataType dataType, Map<String, EsField> properties, boolean hasDocValues) {
@@ -28,7 +28,7 @@ private DateEsField(String name, DataType dataType, Map<String, EsField> propert
     }
 
     private DateEsField(StreamInput in) throws IOException {
-        this(in.readString(), DataTypes.DATETIME, in.readMap(i -> i.readNamedWriteable(EsField.class)), in.readBoolean());
+        this(in.readString(), DataType.DATETIME, in.readMap(i -> i.readNamedWriteable(EsField.class)), in.readBoolean());
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DefaultDataTypeRegistry.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DefaultDataTypeRegistry.java
index cba409236e8c9..9bba9698faff7 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DefaultDataTypeRegistry.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DefaultDataTypeRegistry.java
@@ -19,22 +19,22 @@ private DefaultDataTypeRegistry() {}
 
     @Override
     public Collection<DataType> dataTypes() {
-        return DataTypes.types();
+        return DataType.types();
     }
 
     @Override
     public DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType) {
-        return DataTypes.fromEs(typeName);
+        return DataType.fromEs(typeName);
     }
 
     @Override
     public DataType fromJava(Object value) {
-        return DataTypes.fromJava(value);
+        return DataType.fromJava(value);
     }
 
     @Override
     public boolean isUnsupported(DataType type) {
-        return DataTypes.isUnsupported(type);
+        return DataType.isUnsupported(type);
     }
 
     @Override
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
index 3f9a1415df70c..fd7bfbec4730f 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java
@@ -31,7 +31,7 @@ public class InvalidMappedField extends EsField {
     private final String errorMessage;
 
     public InvalidMappedField(String name, String errorMessage, Map<String, EsField> properties) {
-        super(name, DataTypes.UNSUPPORTED, properties, false);
+        super(name, DataType.UNSUPPORTED, properties, false);
         this.errorMessage = errorMessage;
     }
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
index 975bd75f91f18..d856e3d9d8297 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java
@@ -15,7 +15,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 
 /**
  * Information about a field in an ES index with the {@code keyword} type.
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
index ba5295f1f9a8a..c52230fa65829 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java
@@ -16,8 +16,8 @@
 import java.util.Map;
 import java.util.function.Function;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
 
 /**
  * Information about a field in an es index with the {@code text} type.
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/Types.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/Types.java
index a5a457207ba84..5daa2e0050543 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/Types.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/Types.java
@@ -16,12 +16,12 @@
 import java.util.Map.Entry;
 
 import static java.util.Collections.emptyMap;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NESTED;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.OBJECT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSUPPORTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
 
 public abstract class Types {
 
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
index af1d0d3659bc0..13e4d6ad953a8 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java
@@ -34,7 +34,7 @@ public UnsupportedEsField(String name, String originalType) {
     }
 
     public UnsupportedEsField(String name, String originalType, String inherited, Map<String, EsField> properties) {
-        super(name, DataTypes.UNSUPPORTED, properties, false);
+        super(name, DataType.UNSUPPORTED, properties, false);
         this.originalType = originalType;
         this.inherited = inherited;
     }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java
index 7e8ba58285fd3..c077af4026974 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 
 import java.util.Collection;
 import java.util.List;
@@ -69,8 +69,8 @@ public void testAttributeMapWithSameAliasesCanResolveAttributes() {
     public void testResolve() {
         AttributeMap.Builder<Object> builder = AttributeMap.builder();
         Attribute one = a("one");
-        Attribute two = fieldAttribute("two", DataTypes.INTEGER);
-        Attribute three = fieldAttribute("three", DataTypes.INTEGER);
+        Attribute two = fieldAttribute("two", DataType.INTEGER);
+        Attribute three = fieldAttribute("three", DataType.INTEGER);
         Alias threeAlias = new Alias(Source.EMPTY, "three_alias", three);
         Alias threeAliasAlias = new Alias(Source.EMPTY, "three_alias_alias", threeAlias);
         builder.put(one, of("one"));
@@ -94,8 +94,8 @@ public void testResolve() {
 
     public void testResolveOneHopCycle() {
         AttributeMap.Builder<Object> builder = AttributeMap.builder();
-        Attribute a = fieldAttribute("a", DataTypes.INTEGER);
-        Attribute b = fieldAttribute("b", DataTypes.INTEGER);
+        Attribute a = fieldAttribute("a", DataType.INTEGER);
+        Attribute b = fieldAttribute("b", DataType.INTEGER);
         builder.put(a, a);
         builder.put(b, a);
         AttributeMap<Object> map = builder.build();
@@ -107,10 +107,10 @@ public void testResolveOneHopCycle() {
 
     public void testResolveMultiHopCycle() {
         AttributeMap.Builder<Object> builder = AttributeMap.builder();
-        Attribute a = fieldAttribute("a", DataTypes.INTEGER);
-        Attribute b = fieldAttribute("b", DataTypes.INTEGER);
-        Attribute c = fieldAttribute("c", DataTypes.INTEGER);
-        Attribute d = fieldAttribute("d", DataTypes.INTEGER);
+        Attribute a = fieldAttribute("a", DataType.INTEGER);
+        Attribute b = fieldAttribute("b", DataType.INTEGER);
+        Attribute c = fieldAttribute("c", DataType.INTEGER);
+        Attribute d = fieldAttribute("d", DataType.INTEGER);
         builder.put(a, b);
         builder.put(b, c);
         builder.put(c, d);
@@ -124,7 +124,7 @@ public void testResolveMultiHopCycle() {
 
     private Alias createIntParameterAlias(int index, int value) {
         Source source = new Source(1, index * 5, "?");
-        Literal literal = new Literal(source, value, DataTypes.INTEGER);
+        Literal literal = new Literal(source, value, DataType.INTEGER);
         Alias alias = new Alias(literal.source(), literal.source().text(), literal);
         return alias;
     }
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/LiteralTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/LiteralTests.java
index 32a953165b53c..7e57e8f358ae1 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/LiteralTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/LiteralTests.java
@@ -21,14 +21,14 @@
 import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BYTE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.SHORT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT;
 
 public class LiteralTests extends AbstractNodeTestCase<Literal, Expression> {
     static class ValueAndCompatibleTypes {
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/NullabilityTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/NullabilityTests.java
index 53132fe1c4b8a..fbeac1748ac81 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/NullabilityTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/NullabilityTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import static java.util.Arrays.asList;
 import static org.elasticsearch.xpack.esql.core.expression.Nullability.FALSE;
@@ -36,7 +35,7 @@ public Nullability nullable() {
 
         @Override
         public DataType dataType() {
-            return DataTypes.BOOLEAN;
+            return DataType.BOOLEAN;
         }
 
         @Override
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/FunctionTestUtils.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/FunctionTestUtils.java
index eba95df9d6dd5..8f0ff30074b83 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/FunctionTestUtils.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/FunctionTestUtils.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.time.Instant;
 import java.time.ZonedDateTime;
@@ -18,15 +17,15 @@
 import java.util.Iterator;
 
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 
 public final class FunctionTestUtils {
 
     public static Literal l(Object value) {
-        return new Literal(EMPTY, value, DataTypes.fromJava(value));
+        return new Literal(EMPTY, value, DataType.fromJava(value));
     }
 
     public static Literal l(Object value, DataType type) {
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java
index f7e650b7727b7..6009ca774f8cd 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java
@@ -17,15 +17,15 @@
 import java.util.Arrays;
 
 import static org.elasticsearch.xpack.esql.core.expression.function.scalar.FunctionTestUtils.l;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.SHORT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 
 public class RangeTests extends ESTestCase {
 
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
index 440e48b30536b..84586ed851824 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
@@ -56,7 +56,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
 import java.time.ZoneId;
@@ -83,10 +82,10 @@
 import static org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.PropagateNullable;
 import static org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.ReplaceRegexMatch;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 import static org.hamcrest.Matchers.contains;
 
 public class OptimizerRulesTests extends ESTestCase {
@@ -1399,7 +1398,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() {
 
     // a == 1 AND a == 2 -> nop for date/time fields
     public void testPropagateEquals_ignoreDateTimeFields() {
-        FieldAttribute fa = TestUtils.getFieldAttribute("a", DataTypes.DATETIME);
+        FieldAttribute fa = TestUtils.getFieldAttribute("a", DataType.DATETIME);
         Equals eq1 = equalsOf(fa, ONE);
         Equals eq2 = equalsOf(fa, TWO);
         And and = new And(EMPTY, eq1, eq2);
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQueryTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQueryTests.java
index 47fa87bf188af..47c471af1051c 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQueryTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQueryTests.java
@@ -23,7 +23,7 @@
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 import static org.hamcrest.Matchers.equalTo;
 
 public class MatchQueryTests extends ESTestCase {
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DataTypeConversionTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DataTypeConversionTests.java
index 80754857d7f24..dee41e089de13 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DataTypeConversionTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/DataTypeConversionTests.java
@@ -17,23 +17,23 @@
 import java.math.BigInteger;
 import java.time.ZonedDateTime;
 
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.commonType;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.converterFor;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BYTE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NULL;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.SHORT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSUPPORTED;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
 import static org.elasticsearch.xpack.esql.core.type.DateUtils.asDateTime;
 
 public class DataTypeConversionTests extends ESTestCase {
@@ -539,8 +539,8 @@ public void testCommonType() {
     }
 
     public void testEsDataTypes() {
-        for (DataType type : DataTypes.types()) {
-            assertEquals(type, DataTypes.fromTypeName(type.typeName()));
+        for (DataType type : DataType.types()) {
+            assertEquals(type, DataType.fromTypeName(type.typeName()));
         }
     }
 
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
index 75921778d5970..e72ae0c5c0cda 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/EsFieldTests.java
@@ -12,7 +12,7 @@
 public class EsFieldTests extends AbstractEsFieldTypeTests<EsField> {
     static EsField randomEsField(int maxPropertiesDepth) {
         String name = randomAlphaOfLength(4);
-        DataType esDataType = randomFrom(DataTypes.types());
+        DataType esDataType = randomFrom(DataType.types());
         Map<String, EsField> properties = randomProperties(maxPropertiesDepth);
         boolean aggregatable = randomBoolean();
         boolean isAlias = randomBoolean();
@@ -33,7 +33,7 @@ protected EsField mutate(EsField instance) {
         boolean isAlias = instance.isAlias();
         switch (between(0, 4)) {
             case 0 -> name = randomAlphaOfLength(name.length() + 1);
-            case 1 -> esDataType = randomValueOtherThan(esDataType, () -> randomFrom(DataTypes.types()));
+            case 1 -> esDataType = randomValueOtherThan(esDataType, () -> randomFrom(DataType.types()));
             case 2 -> properties = randomValueOtherThan(properties, () -> randomProperties(4));
             case 3 -> aggregatable = false == aggregatable;
             case 4 -> isAlias = false == isAlias;
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java
index b3be18156ab8b..489666976b592 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/type/TypesTests.java
@@ -15,12 +15,12 @@
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NESTED;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.OBJECT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 
@@ -128,7 +128,7 @@ public void testDottedField() {
 
         assertThat(mapping.size(), is(2));
         EsField field = mapping.get("manager");
-        assertThat(DataTypes.isPrimitive(field.getDataType()), is(false));
+        assertThat(DataType.isPrimitive(field.getDataType()), is(false));
         assertThat(field.getDataType(), is(OBJECT));
         Map<String, EsField> children = field.getProperties();
         assertThat(children.size(), is(2));
@@ -143,7 +143,7 @@ public void testMultiField() {
 
         assertThat(mapping.size(), is(1));
         EsField field = mapping.get("text");
-        assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
+        assertThat(DataType.isPrimitive(field.getDataType()), is(true));
         assertThat(field.getDataType(), is(TEXT));
         Map<String, EsField> fields = field.getProperties();
         assertThat(fields.size(), is(4));
@@ -157,7 +157,7 @@ public void testMultiFieldTooManyOptions() {
 
         assertThat(mapping.size(), is(1));
         EsField field = mapping.get("text");
-        assertThat(DataTypes.isPrimitive(field.getDataType()), is(true));
+        assertThat(DataType.isPrimitive(field.getDataType()), is(true));
         assertThat(field, instanceOf(TextEsField.class));
         Map<String, EsField> fields = field.getProperties();
         assertThat(fields.size(), is(4));
@@ -171,7 +171,7 @@ public void testNestedDoc() {
 
         assertThat(mapping.size(), is(1));
         EsField field = mapping.get("dep");
-        assertThat(DataTypes.isPrimitive(field.getDataType()), is(false));
+        assertThat(DataType.isPrimitive(field.getDataType()), is(false));
         assertThat(field.getDataType(), is(NESTED));
         Map<String, EsField> children = field.getProperties();
         assertThat(children.size(), is(4));
diff --git a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java b/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java
index 27da0f56deb20..35d73f87f2ceb 100644
--- a/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java
+++ b/x-pack/plugin/esql-core/test-fixtures/src/main/java/org/elasticsearch/xpack/esql/core/TestUtils.java
@@ -41,7 +41,6 @@
 import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.hamcrest.Description;
@@ -82,7 +81,7 @@
 import static org.elasticsearch.test.ESTestCase.randomZone;
 import static org.elasticsearch.xpack.esql.core.TestUtils.StringContainsRegex.containsRegex;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 import static org.hamcrest.Matchers.containsString;
 import static org.junit.Assert.assertEquals;
 
@@ -115,7 +114,7 @@ public static Literal of(Source source, Object value) {
         if (value instanceof Literal) {
             return (Literal) value;
         }
-        return new Literal(source, value, DataTypes.fromJava(value));
+        return new Literal(source, value, DataType.fromJava(value));
     }
 
     public static Equals equalsOf(Expression left, Expression right) {
@@ -159,7 +158,7 @@ public static RLike rlike(Expression left, String exp) {
     }
 
     public static FieldAttribute fieldAttribute() {
-        return fieldAttribute(randomAlphaOfLength(10), randomFrom(DataTypes.types()));
+        return fieldAttribute(randomAlphaOfLength(10), randomFrom(DataType.types()));
     }
 
     public static FieldAttribute fieldAttribute(String name, DataType type) {
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
index 820d62eb50e37..86b931046e6db 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.DateUtils;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.type.TypesTests;
@@ -300,11 +299,11 @@ static Map<String, Map<String, Column>> tables() {
             BytesRefBlock namesBlock = names.build();
             tables.put(
                 "int_number_names",
-                Map.of("int", new Column(DataTypes.INTEGER, intsBlock), "name", new Column(DataTypes.KEYWORD, namesBlock))
+                Map.of("int", new Column(DataType.INTEGER, intsBlock), "name", new Column(DataType.KEYWORD, namesBlock))
             );
             tables.put(
                 "long_number_names",
-                Map.of("long", new Column(DataTypes.LONG, longsBlock), "name", new Column(DataTypes.KEYWORD, namesBlock))
+                Map.of("long", new Column(DataType.LONG, longsBlock), "name", new Column(DataType.KEYWORD, namesBlock))
             );
         }
         return unmodifiableMap(tables);
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java
index 174df48d5ce62..57427f87b76f7 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java
@@ -42,12 +42,12 @@ public IpPrefixEvaluator(Source source, EvalOperator.ExpressionEvaluator ip,
       EvalOperator.ExpressionEvaluator prefixLengthV4,
       EvalOperator.ExpressionEvaluator prefixLengthV6, BytesRef scratch,
       DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.ip = ip;
     this.prefixLengthV4 = prefixLengthV4;
     this.prefixLengthV6 = prefixLengthV6;
     this.scratch = scratch;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ParseTables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ParseTables.java
index 1c607e6446f4c..c9748ee214820 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ParseTables.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ParseTables.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.esql.Column;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.io.IOException;
 import java.util.Map;
@@ -32,7 +31,7 @@
  * Parses the {@code tables} request body parameter.
  */
 public class ParseTables {
-    public static final Set<DataType> SUPPORTED_TYPES = Set.of(DataTypes.INTEGER, DataTypes.KEYWORD, DataTypes.LONG);
+    public static final Set<DataType> SUPPORTED_TYPES = Set.of(DataType.INTEGER, DataType.KEYWORD, DataType.LONG);
     private static final int MAX_LENGTH = (int) ByteSizeValue.ofMb(1).getBytes();
 
     private final BlockFactory blockFactory;
@@ -127,7 +126,7 @@ private Column parseKeywordColumn() throws IOException {
             while (true) {
                 switch (p.nextToken()) {
                     case END_ARRAY -> {
-                        return new Column(DataTypes.KEYWORD, builder.build());
+                        return new Column(DataType.KEYWORD, builder.build());
                     }
                     case START_ARRAY -> parseTextArray(builder, scratch);
                     case VALUE_NULL -> builder.appendNull();
@@ -172,7 +171,7 @@ private Column parseIntColumn() throws IOException {
             while (true) {
                 switch (p.nextToken()) {
                     case END_ARRAY -> {
-                        return new Column(DataTypes.INTEGER, builder.build());
+                        return new Column(DataType.INTEGER, builder.build());
                     }
                     case START_ARRAY -> parseIntArray(builder);
                     case VALUE_NULL -> builder.appendNull();
@@ -214,7 +213,7 @@ private Column parseLongColumn() throws IOException {
             while (true) {
                 switch (p.nextToken()) {
                     case END_ARRAY -> {
-                        return new Column(DataTypes.LONG, builder.build());
+                        return new Column(DataType.LONG, builder.build());
                     }
                     case START_ARRAY -> parseLongArray(builder);
                     case VALUE_NULL -> builder.appendNull();
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index b95afe2fadeea..90b2197f11dae 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -46,7 +46,6 @@
 import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
 import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
@@ -94,26 +93,26 @@
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.GEO_MATCH_TYPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NESTED;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.stats.FeatureMetric.LIMIT;
 
 public class Analyzer extends ParameterizedRuleExecutor<LogicalPlan, AnalyzerContext> {
     // marker list of attributes for plans that do not have any concrete fields to return, but have other computed columns to return
     // ie from test | stats c = count(*)
     public static final List<Attribute> NO_FIELDS = List.of(
-        new ReferenceAttribute(Source.EMPTY, "<no-fields>", DataTypes.NULL, null, Nullability.TRUE, null, true)
+        new ReferenceAttribute(Source.EMPTY, "<no-fields>", DataType.NULL, null, Nullability.TRUE, null, true)
     );
     private static final Iterable<RuleExecutor.Batch<LogicalPlan>> rules;
 
@@ -777,7 +776,7 @@ public LogicalPlan apply(LogicalPlan logicalPlan, AnalyzerContext context) {
                 limit = context.configuration().resultTruncationMaxSize(); // user provided a limit: cap result entries to the max
             }
             var source = logicalPlan.source();
-            return new Limit(source, new Literal(source, limit, DataTypes.INTEGER), logicalPlan);
+            return new Limit(source, new Literal(source, limit, DataType.INTEGER), logicalPlan);
         }
     }
 
@@ -820,7 +819,7 @@ private static Expression processScalarFunction(EsqlScalarFunction f, EsqlFuncti
             }
             List<Expression> newChildren = new ArrayList<>(args.size());
             boolean childrenChanged = false;
-            DataType targetDataType = DataTypes.NULL;
+            DataType targetDataType = DataType.NULL;
             Expression arg;
             for (int i = 0; i < args.size(); i++) {
                 arg = args.get(i);
@@ -828,7 +827,7 @@ private static Expression processScalarFunction(EsqlScalarFunction f, EsqlFuncti
                     if (i < targetDataTypes.size()) {
                         targetDataType = targetDataTypes.get(i);
                     }
-                    if (targetDataType != DataTypes.NULL && targetDataType != DataTypes.UNSUPPORTED) {
+                    if (targetDataType != DataType.NULL && targetDataType != DataType.UNSUPPORTED) {
                         Expression e = castStringLiteral(arg, targetDataType);
                         childrenChanged = true;
                         newChildren.add(e);
@@ -848,7 +847,7 @@ private static Expression processBinaryOperator(BinaryOperator<?, ?, ?, ?> o) {
             }
             List<Expression> newChildren = new ArrayList<>(2);
             boolean childrenChanged = false;
-            DataType targetDataType = DataTypes.NULL;
+            DataType targetDataType = DataType.NULL;
             Expression from = Literal.NULL;
 
             if (left.dataType() == KEYWORD
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
index 3f6e387e661f1..87fcf05383128 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction;
@@ -378,17 +377,17 @@ public static Failure validateBinaryComparison(BinaryComparison bc) {
         }
 
         List<DataType> allowed = new ArrayList<>();
-        allowed.add(DataTypes.KEYWORD);
-        allowed.add(DataTypes.TEXT);
-        allowed.add(DataTypes.IP);
-        allowed.add(DataTypes.DATETIME);
-        allowed.add(DataTypes.VERSION);
-        allowed.add(DataTypes.GEO_POINT);
-        allowed.add(DataTypes.GEO_SHAPE);
-        allowed.add(DataTypes.CARTESIAN_POINT);
-        allowed.add(DataTypes.CARTESIAN_SHAPE);
+        allowed.add(DataType.KEYWORD);
+        allowed.add(DataType.TEXT);
+        allowed.add(DataType.IP);
+        allowed.add(DataType.DATETIME);
+        allowed.add(DataType.VERSION);
+        allowed.add(DataType.GEO_POINT);
+        allowed.add(DataType.GEO_SHAPE);
+        allowed.add(DataType.CARTESIAN_POINT);
+        allowed.add(DataType.CARTESIAN_SHAPE);
         if (bc instanceof Equals || bc instanceof NotEquals) {
-            allowed.add(DataTypes.BOOLEAN);
+            allowed.add(DataType.BOOLEAN);
         }
         Expression.TypeResolution r = TypeResolutions.isType(
             bc.left(),
@@ -400,7 +399,7 @@ public static Failure validateBinaryComparison(BinaryComparison bc) {
         if (false == r.resolved()) {
             return fail(bc, r.message());
         }
-        if (DataTypes.isString(bc.left().dataType()) && DataTypes.isString(bc.right().dataType())) {
+        if (DataType.isString(bc.left().dataType()) && DataType.isString(bc.right().dataType())) {
             return null;
         }
         if (bc.left().dataType() != bc.right().dataType()) {
@@ -427,15 +426,15 @@ public static Failure validateBinaryComparison(BinaryComparison bc) {
     public static Failure validateUnsignedLongOperator(BinaryOperator<?, ?, ?, ?> bo) {
         DataType leftType = bo.left().dataType();
         DataType rightType = bo.right().dataType();
-        if ((leftType == DataTypes.UNSIGNED_LONG || rightType == DataTypes.UNSIGNED_LONG) && leftType != rightType) {
+        if ((leftType == DataType.UNSIGNED_LONG || rightType == DataType.UNSIGNED_LONG) && leftType != rightType) {
             return fail(
                 bo,
                 "first argument of [{}] is [{}] and second is [{}]. [{}] can only be operated on together with another [{}]",
                 bo.sourceText(),
                 leftType.typeName(),
                 rightType.typeName(),
-                DataTypes.UNSIGNED_LONG.typeName(),
-                DataTypes.UNSIGNED_LONG.typeName()
+                DataType.UNSIGNED_LONG.typeName(),
+                DataType.UNSIGNED_LONG.typeName()
             );
         }
         return null;
@@ -446,7 +445,7 @@ public static Failure validateUnsignedLongOperator(BinaryOperator<?, ?, ?, ?> bo
      */
     private static Failure validateUnsignedLongNegation(Neg neg) {
         DataType childExpressionType = neg.field().dataType();
-        if (childExpressionType.equals(DataTypes.UNSIGNED_LONG)) {
+        if (childExpressionType.equals(DataType.UNSIGNED_LONG)) {
             return fail(
                 neg,
                 "negation unsupported for arguments of type [{}] in expression [{}]",
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/QueryList.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/QueryList.java
index 929206ed58897..923b75055ca9d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/QueryList.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/QueryList.java
@@ -33,8 +33,8 @@
 import java.util.List;
 import java.util.function.IntFunction;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
 
 /**
  * Generates a list of Lucene queries based on the input block.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java
index 1e2e58886792a..8f7fcef0ff07e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/EsqlTypeResolutions.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
@@ -20,10 +19,10 @@
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
 
 public class EsqlTypeResolutions {
 
@@ -38,7 +37,7 @@ public static Expression.TypeResolution isStringAndExact(Expression e, String op
 
     public static Expression.TypeResolution isExact(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
         if (e instanceof FieldAttribute fa) {
-            if (DataTypes.isString(fa.dataType())) {
+            if (DataType.isString(fa.dataType())) {
                 // ESQL can extract exact values for TEXT fields
                 return Expression.TypeResolution.TYPE_RESOLVED;
             }
@@ -65,7 +64,7 @@ public static Expression.TypeResolution isExact(Expression e, String operationNa
         GEO_SHAPE.typeName(),
         CARTESIAN_SHAPE.typeName() };
     private static final String[] POINT_TYPE_NAMES = new String[] { GEO_POINT.typeName(), CARTESIAN_POINT.typeName() };
-    private static final String[] NON_SPATIAL_TYPE_NAMES = DataTypes.types()
+    private static final String[] NON_SPATIAL_TYPE_NAMES = DataType.types()
         .stream()
         .filter(EsqlDataTypes::isRepresentable)
         .filter(t -> EsqlDataTypes.isSpatial(t) == false)
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java
index fa66f299b2272..10800a2394e8f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 
 import java.util.List;
 
@@ -21,7 +21,7 @@ public Order(Source source, Expression child, OrderDirection direction, NullsPos
 
     @Override
     protected TypeResolution resolveType() {
-        if (DataTypes.isString(child().dataType())) {
+        if (DataType.isString(child().dataType())) {
             return TypeResolution.TYPE_RESOLVED;
         }
         return super.resolveType();
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index af287399a8ddb..aa335a1d09400 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.esql.core.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.esql.core.session.Configuration;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Avg;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.CountDistinct;
@@ -124,21 +123,21 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSUPPORTED;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 
 public final class EsqlFunctionRegistry extends FunctionRegistry {
 
@@ -376,7 +375,7 @@ public List<String> argDescriptions() {
     public static DataType getTargetType(String[] names) {
         List<DataType> types = new ArrayList<>();
         for (String name : names) {
-            types.add(DataTypes.fromEs(name));
+            types.add(DataType.fromEs(name));
         }
         if (types.contains(KEYWORD) || types.contains(TEXT)) {
             return UNSUPPORTED;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java
index 39d73dd82e39c..ee75980e10264 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -34,7 +33,7 @@ public Avg(Source source, @Param(name = "number", type = { "double", "integer",
     protected Expression.TypeResolution resolveType() {
         return isType(
             field(),
-            dt -> dt.isNumeric() && dt != DataTypes.UNSIGNED_LONG,
+            dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG,
             sourceText(),
             DEFAULT,
             "numeric except unsigned_long or counter types"
@@ -43,7 +42,7 @@ protected Expression.TypeResolution resolveType() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java
index 73bbfba544bd9..d55bc9d618c39 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -75,7 +74,7 @@ public String innerName() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.LONG;
+        return DataType.LONG;
     }
 
     @Override
@@ -103,7 +102,7 @@ public Expression surrogate() {
                 if (l.value() != null && (l.value() instanceof List<?>) == false) {
                     // TODO: Normalize COUNT(*), COUNT(), COUNT("foobar"), COUNT(1) as COUNT(*).
                     // Does not apply to COUNT([1,2,3])
-                    // return new Count(s, new Literal(s, StringUtils.WILDCARD, DataTypes.KEYWORD));
+                    // return new Count(s, new Literal(s, StringUtils.WILDCARD, DataType.KEYWORD));
                     return null;
                 }
             }
@@ -111,8 +110,8 @@ public Expression surrogate() {
             // COUNT(const) is equivalent to MV_COUNT(const)*COUNT(*) if const is not null; otherwise COUNT(const) == 0.
             return new Mul(
                 s,
-                new Coalesce(s, new MvCount(s, field), List.of(new Literal(s, 0, DataTypes.INTEGER))),
-                new Count(s, new Literal(s, StringUtils.WILDCARD, DataTypes.KEYWORD))
+                new Coalesce(s, new MvCount(s, field), List.of(new Literal(s, 0, DataType.INTEGER))),
+                new Count(s, new Literal(s, StringUtils.WILDCARD, DataType.KEYWORD))
             );
         }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
index 481f025399842..c91b9c37ae0a3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -69,7 +68,7 @@ public CountDistinct replaceChildren(List<Expression> newChildren) {
 
     @Override
     public DataType dataType() {
-        return DataTypes.LONG;
+        return DataType.LONG;
     }
 
     @Override
@@ -86,7 +85,7 @@ protected TypeResolution resolveType() {
         boolean resolved = resolution.resolved();
         resolution = isType(
             field(),
-            dt -> resolved && dt != DataTypes.UNSIGNED_LONG,
+            dt -> resolved && dt != DataType.UNSIGNED_LONG,
             sourceText(),
             DEFAULT,
             "any exact type except unsigned_long or counter types"
@@ -101,20 +100,20 @@ protected TypeResolution resolveType() {
     public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         DataType type = field().dataType();
         int precision = this.precision == null ? DEFAULT_PRECISION : ((Number) this.precision.fold()).intValue();
-        if (type == DataTypes.BOOLEAN) {
+        if (type == DataType.BOOLEAN) {
             // Booleans ignore the precision because there are only two possible values anyway
             return new CountDistinctBooleanAggregatorFunctionSupplier(inputChannels);
         }
-        if (type == DataTypes.DATETIME || type == DataTypes.LONG) {
+        if (type == DataType.DATETIME || type == DataType.LONG) {
             return new CountDistinctLongAggregatorFunctionSupplier(inputChannels, precision);
         }
-        if (type == DataTypes.INTEGER) {
+        if (type == DataType.INTEGER) {
             return new CountDistinctIntAggregatorFunctionSupplier(inputChannels, precision);
         }
-        if (type == DataTypes.DOUBLE) {
+        if (type == DataType.DOUBLE) {
             return new CountDistinctDoubleAggregatorFunctionSupplier(inputChannels, precision);
         }
-        if (type == DataTypes.KEYWORD || type == DataTypes.IP || type == DataTypes.VERSION || type == DataTypes.TEXT) {
+        if (type == DataType.KEYWORD || type == DataType.IP || type == DataType.VERSION || type == DataType.TEXT) {
             return new CountDistinctBytesRefAggregatorFunctionSupplier(inputChannels, precision);
         }
         throw EsqlIllegalArgumentException.illegalDataType(type);
@@ -126,7 +125,7 @@ public Expression surrogate() {
         var field = field();
 
         return field.foldable()
-            ? new ToLong(s, new Coalesce(s, new MvCount(s, new MvDedupe(s, field)), List.of(new Literal(s, 0, DataTypes.INTEGER))))
+            ? new ToLong(s, new Coalesce(s, new MvCount(s, new MvDedupe(s, field)), List.of(new Literal(s, 0, DataType.INTEGER))))
             : null;
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java
index 7c48a58ddb649..c381693dbe2ce 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -40,7 +39,7 @@ public Median(Source source, @Param(name = "number", type = { "double", "integer
     protected Expression.TypeResolution resolveType() {
         return isType(
             field(),
-            dt -> dt.isNumeric() && dt != DataTypes.UNSIGNED_LONG,
+            dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG,
             sourceText(),
             DEFAULT,
             "numeric except unsigned_long or counter types"
@@ -49,7 +48,7 @@ protected Expression.TypeResolution resolveType() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
@@ -69,6 +68,6 @@ public Expression surrogate() {
 
         return field.foldable()
             ? new MvMedian(s, new ToDouble(s, field))
-            : new Percentile(source(), field(), new Literal(source(), (int) QuantileStates.MEDIAN, DataTypes.INTEGER));
+            : new Percentile(source(), field(), new Literal(source(), (int) QuantileStates.MEDIAN, DataType.INTEGER));
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java
index 7f77f9cbb8ccb..b003b981c0709 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.planner.ToAggregator;
 
 import java.util.List;
@@ -35,7 +34,7 @@ protected TypeResolution resolveType() {
         if (supportsDates()) {
             return TypeResolutions.isType(
                 this,
-                e -> e == DataTypes.DATETIME || e.isNumeric() && e != DataTypes.UNSIGNED_LONG,
+                e -> e == DataType.DATETIME || e.isNumeric() && e != DataType.UNSIGNED_LONG,
                 sourceText(),
                 DEFAULT,
                 "datetime",
@@ -44,7 +43,7 @@ protected TypeResolution resolveType() {
         }
         return isType(
             field(),
-            dt -> dt.isNumeric() && dt != DataTypes.UNSIGNED_LONG,
+            dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG,
             sourceText(),
             DEFAULT,
             "numeric except unsigned_long or counter types"
@@ -57,22 +56,22 @@ protected boolean supportsDates() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
     public final AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         DataType type = field().dataType();
-        if (supportsDates() && type == DataTypes.DATETIME) {
+        if (supportsDates() && type == DataType.DATETIME) {
             return longSupplier(inputChannels);
         }
-        if (type == DataTypes.LONG) {
+        if (type == DataType.LONG) {
             return longSupplier(inputChannels);
         }
-        if (type == DataTypes.INTEGER) {
+        if (type == DataType.INTEGER) {
             return intSupplier(inputChannels);
         }
-        if (type == DataTypes.DOUBLE) {
+        if (type == DataType.DOUBLE) {
             return doubleSupplier(inputChannels);
         }
         throw EsqlIllegalArgumentException.illegalDataType(type);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
index b06053199c5ab..d21247a77d9cf 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 
@@ -65,7 +65,7 @@ protected TypeResolution resolveType() {
 
         TypeResolution resolution = isType(
             field(),
-            dt -> dt.isNumeric() && dt != DataTypes.UNSIGNED_LONG,
+            dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG,
             sourceText(),
             FIRST,
             "numeric except unsigned_long"
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
index 0c5db63209b33..418f92284cca0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SpatialCentroid.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.planner.ToAggregator;
@@ -72,18 +71,18 @@ public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         DataType type = field().dataType();
         if (useDocValues) {
             // When the points are read as doc-values (eg. from the index), feed them into the doc-values aggregator
-            if (type == DataTypes.GEO_POINT) {
+            if (type == DataType.GEO_POINT) {
                 return new SpatialCentroidGeoPointDocValuesAggregatorFunctionSupplier(inputChannels);
             }
-            if (type == DataTypes.CARTESIAN_POINT) {
+            if (type == DataType.CARTESIAN_POINT) {
                 return new SpatialCentroidCartesianPointDocValuesAggregatorFunctionSupplier(inputChannels);
             }
         } else {
             // When the points are read as WKB from source or as point literals, feed them into the source-values aggregator
-            if (type == DataTypes.GEO_POINT) {
+            if (type == DataType.GEO_POINT) {
                 return new SpatialCentroidGeoPointSourceValuesAggregatorFunctionSupplier(inputChannels);
             }
-            if (type == DataTypes.CARTESIAN_POINT) {
+            if (type == DataType.CARTESIAN_POINT) {
                 return new SpatialCentroidCartesianPointSourceValuesAggregatorFunctionSupplier(inputChannels);
             }
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java
index 78ad04414cd57..be9ae295f6fbc 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -25,9 +24,9 @@
 
 import java.util.List;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 
 /**
  * Sum all values of a field in matching documents.
@@ -77,7 +76,7 @@ public Expression surrogate() {
 
         // SUM(const) is equivalent to MV_SUM(const)*COUNT(*).
         return field.foldable()
-            ? new Mul(s, new MvSum(s, field), new Count(s, new Literal(s, StringUtils.WILDCARD, DataTypes.KEYWORD)))
+            ? new Mul(s, new MvSum(s, field), new Count(s, new Literal(s, StringUtils.WILDCARD, DataType.KEYWORD)))
             : null;
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
index 1723027a8e8e0..c76f60fe0f555 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -64,19 +63,19 @@ protected TypeResolution resolveType() {
     @Override
     public AggregatorFunctionSupplier supplier(List<Integer> inputChannels) {
         DataType type = field().dataType();
-        if (type == DataTypes.INTEGER) {
+        if (type == DataType.INTEGER) {
             return new ValuesIntAggregatorFunctionSupplier(inputChannels);
         }
-        if (type == DataTypes.LONG || type == DataTypes.DATETIME) {
+        if (type == DataType.LONG || type == DataType.DATETIME) {
             return new ValuesLongAggregatorFunctionSupplier(inputChannels);
         }
-        if (type == DataTypes.DOUBLE) {
+        if (type == DataType.DOUBLE) {
             return new ValuesDoubleAggregatorFunctionSupplier(inputChannels);
         }
-        if (DataTypes.isString(type) || type == DataTypes.IP || type == DataTypes.VERSION) {
+        if (DataType.isString(type) || type == DataType.IP || type == DataType.VERSION) {
             return new ValuesBytesRefAggregatorFunctionSupplier(inputChannels);
         }
-        if (type == DataTypes.BOOLEAN) {
+        if (type == DataType.BOOLEAN) {
             return new ValuesBooleanAggregatorFunctionSupplier(inputChannels);
         }
         // TODO cartesian_point, geo_point
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
index 6e8341614a973..431494534f4ec 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -201,7 +200,7 @@ public boolean foldable() {
 
     @Override
     public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
-        if (field.dataType() == DataTypes.DATETIME) {
+        if (field.dataType() == DataType.DATETIME) {
             Rounding.Prepared preparedRounding;
             if (buckets.dataType().isInteger()) {
                 int b = ((Number) buckets.fold()).intValue();
@@ -225,7 +224,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
                 assert buckets.dataType().isRational() : "Unexpected rounding data type [" + buckets.dataType() + "]";
                 roundTo = ((Number) buckets.fold()).doubleValue();
             }
-            Literal rounding = new Literal(source(), roundTo, DataTypes.DOUBLE);
+            Literal rounding = new Literal(source(), roundTo, DataType.DOUBLE);
 
             // We could make this more efficient, either by generating the evaluators with byte code or hand rolling this one.
             Div div = new Div(source(), field, rounding);
@@ -286,11 +285,11 @@ protected TypeResolution resolveType() {
         }
         var fieldType = field.dataType();
         var bucketsType = buckets.dataType();
-        if (fieldType == DataTypes.NULL || bucketsType == DataTypes.NULL) {
+        if (fieldType == DataType.NULL || bucketsType == DataType.NULL) {
             return TypeResolution.TYPE_RESOLVED;
         }
 
-        if (fieldType == DataTypes.DATETIME) {
+        if (fieldType == DataType.DATETIME) {
             TypeResolution resolution = isType(
                 buckets,
                 dt -> dt.isInteger() || EsqlDataTypes.isTemporalAmount(dt),
@@ -340,7 +339,7 @@ private TypeResolution checkArgsCount(int expectedCount) {
     private static TypeResolution isStringOrDate(Expression e, String operationName, TypeResolutions.ParamOrdinal paramOrd) {
         return TypeResolutions.isType(
             e,
-            exp -> DataTypes.isString(exp) || DataTypes.isDateTime(exp),
+            exp -> DataType.isString(exp) || DataType.isDateTime(exp),
             operationName,
             paramOrd,
             "datetime",
@@ -359,13 +358,13 @@ public void validate(Failures failures) {
 
     private long foldToLong(Expression e) {
         Object value = Foldables.valueOf(e);
-        return DataTypes.isDateTime(e.dataType()) ? ((Number) value).longValue() : dateTimeToLong(((BytesRef) value).utf8ToString());
+        return DataType.isDateTime(e.dataType()) ? ((Number) value).longValue() : dateTimeToLong(((BytesRef) value).utf8ToString());
     }
 
     @Override
     public DataType dataType() {
         if (field.dataType().isNumeric()) {
-            return DataTypes.DOUBLE;
+            return DataType.DOUBLE;
         }
         return field.dataType();
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java
index f67940e95ec09..f98f5c45acd16 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java
@@ -36,7 +36,7 @@
 import java.util.stream.Stream;
 
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NULL;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
 
 public final class Case extends EsqlScalarFunction {
     record Condition(Expression condition, Expression value) {}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java
index 3491cd00f9ae0..8062019b4c51c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -29,7 +28,7 @@
 import java.util.function.Function;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NULL;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
 
 /**
  * Returns the maximum value of multiple columns.
@@ -118,23 +117,23 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
         ExpressionEvaluator.Factory[] factories = children().stream()
             .map(e -> toEvaluator.apply(new MvMax(e.source(), e)))
             .toArray(ExpressionEvaluator.Factory[]::new);
-        if (dataType == DataTypes.BOOLEAN) {
+        if (dataType == DataType.BOOLEAN) {
             return new GreatestBooleanEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataTypes.DOUBLE) {
+        if (dataType == DataType.DOUBLE) {
             return new GreatestDoubleEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataTypes.INTEGER) {
+        if (dataType == DataType.INTEGER) {
             return new GreatestIntEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataTypes.LONG) {
+        if (dataType == DataType.LONG) {
             return new GreatestLongEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataTypes.KEYWORD
-            || dataType == DataTypes.TEXT
-            || dataType == DataTypes.IP
-            || dataType == DataTypes.VERSION
-            || dataType == DataTypes.UNSUPPORTED) {
+        if (dataType == DataType.KEYWORD
+            || dataType == DataType.TEXT
+            || dataType == DataType.IP
+            || dataType == DataType.VERSION
+            || dataType == DataType.UNSUPPORTED) {
 
             return new GreatestBytesRefEvaluator.Factory(source(), factories);
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java
index 1238418264fbb..f983e0125a4db 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -29,7 +28,7 @@
 import java.util.function.Function;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NULL;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
 
 /**
  * Returns the minimum value of multiple columns.
@@ -117,23 +116,23 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
         ExpressionEvaluator.Factory[] factories = children().stream()
             .map(e -> toEvaluator.apply(new MvMin(e.source(), e)))
             .toArray(ExpressionEvaluator.Factory[]::new);
-        if (dataType == DataTypes.BOOLEAN) {
+        if (dataType == DataType.BOOLEAN) {
             return new LeastBooleanEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataTypes.DOUBLE) {
+        if (dataType == DataType.DOUBLE) {
             return new LeastDoubleEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataTypes.INTEGER) {
+        if (dataType == DataType.INTEGER) {
             return new LeastIntEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataTypes.LONG) {
+        if (dataType == DataType.LONG) {
             return new LeastLongEvaluator.Factory(source(), factories);
         }
-        if (dataType == DataTypes.KEYWORD
-            || dataType == DataTypes.TEXT
-            || dataType == DataTypes.IP
-            || dataType == DataTypes.VERSION
-            || dataType == DataTypes.UNSUPPORTED) {
+        if (dataType == DataType.KEYWORD
+            || dataType == DataType.TEXT
+            || dataType == DataType.IP
+            || dataType == DataType.VERSION
+            || dataType == DataType.UNSUPPORTED) {
 
             return new LeastBytesRefEvaluator.Factory(source(), factories);
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
index 54d9fcb59f476..955ce1646813f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Warnings;
 import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -43,13 +42,8 @@
 public abstract class AbstractConvertFunction extends UnaryScalarFunction {
 
     // the numeric types convert functions need to handle; the other numeric types are converted upstream to one of these
-    private static final List<DataType> NUMERIC_TYPES = List.of(
-        DataTypes.INTEGER,
-        DataTypes.LONG,
-        DataTypes.UNSIGNED_LONG,
-        DataTypes.DOUBLE
-    );
-    public static final List<DataType> STRING_TYPES = DataTypes.types().stream().filter(EsqlDataTypes::isString).toList();
+    private static final List<DataType> NUMERIC_TYPES = List.of(DataType.INTEGER, DataType.LONG, DataType.UNSIGNED_LONG, DataType.DOUBLE);
+    public static final List<DataType> STRING_TYPES = DataType.types().stream().filter(EsqlDataTypes::isString).toList();
 
     protected AbstractConvertFunction(Source source, Expression field) {
         super(source, field);
@@ -88,7 +82,7 @@ public static String supportedTypesNames(Set<DataType> types) {
             STRING_TYPES.forEach(supportTypes::remove);
         }
 
-        supportTypes.forEach(t -> supportedTypesNames.add(t.name().toLowerCase(Locale.ROOT)));
+        supportTypes.forEach(t -> supportedTypesNames.add(t.nameUpper().toLowerCase(Locale.ROOT)));
         supportedTypesNames.sort(String::compareTo);
         return Strings.join(supportedTypesNames, " or ");
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java
index 6cbcd853be3c8..2ebc3d824a4e1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java
@@ -29,7 +29,7 @@
 import java.util.function.Function;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 
 public class FromBase64 extends UnaryScalarFunction {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java
index b2b15a3b824b4..f246513375501 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java
@@ -29,7 +29,7 @@
 import java.util.function.Function;
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 
 public class ToBase64 extends UnaryScalarFunction {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java
index 127ad0d8c471c..5de31b7f114e1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java
@@ -20,13 +20,13 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToBoolean;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToBoolean;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java
index 5c4208b9f0ce8..9a3362c52c0f9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java
@@ -20,9 +20,9 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial;
 
 public class ToCartesianPoint extends AbstractConvertFunction {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java
index df2c5186ab730..03e8c565f342b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java
@@ -20,10 +20,10 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial;
 
 public class ToCartesianShape extends AbstractConvertFunction {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java
index ed14c22092871..53c87427b9607 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java
@@ -20,13 +20,13 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong;
 
 public class ToDatetime extends AbstractConvertFunction {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegrees.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegrees.java
index 590ab9de11d40..af163b6ff8788 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegrees.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegrees.java
@@ -21,10 +21,10 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 
 /**
  * Converts from <a href="https://en.wikipedia.org/wiki/Radian">radians</a>
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java
index 901257b63ec2b..b30162ef23349 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -22,14 +21,14 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToDouble;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToDouble;
 
@@ -44,9 +43,9 @@ public class ToDouble extends AbstractConvertFunction {
         Map.entry(UNSIGNED_LONG, ToDoubleFromUnsignedLongEvaluator.Factory::new),
         Map.entry(LONG, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd
         Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new), // CastIntToDoubleEvaluator would be a candidate, but not MV'd
-        Map.entry(DataTypes.COUNTER_DOUBLE, (field, source) -> field),
-        Map.entry(DataTypes.COUNTER_INTEGER, ToDoubleFromIntEvaluator.Factory::new),
-        Map.entry(DataTypes.COUNTER_LONG, ToDoubleFromLongEvaluator.Factory::new)
+        Map.entry(DataType.COUNTER_DOUBLE, (field, source) -> field),
+        Map.entry(DataType.COUNTER_INTEGER, ToDoubleFromIntEvaluator.Factory::new),
+        Map.entry(DataType.COUNTER_LONG, ToDoubleFromLongEvaluator.Factory::new)
     );
 
     @FunctionInfo(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java
index 122211efbcf10..4692155fc9733 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java
@@ -20,9 +20,9 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial;
 
 public class ToGeoPoint extends AbstractConvertFunction {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java
index 9d7cc08ec6345..93acd578f86c0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java
@@ -20,10 +20,10 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial;
 
 public class ToGeoShape extends AbstractConvertFunction {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java
index b7340f1853d3d..92467d98472c5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java
@@ -20,9 +20,9 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToIP;
 
 public class ToIP extends AbstractConvertFunction {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java
index ea86328a15362..aa926eee3a558 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -22,15 +21,15 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToInt;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToInt;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToInt;
 
@@ -45,7 +44,7 @@ public class ToInteger extends AbstractConvertFunction {
         Map.entry(DOUBLE, ToIntegerFromDoubleEvaluator.Factory::new),
         Map.entry(UNSIGNED_LONG, ToIntegerFromUnsignedLongEvaluator.Factory::new),
         Map.entry(LONG, ToIntegerFromLongEvaluator.Factory::new),
-        Map.entry(DataTypes.COUNTER_INTEGER, (fieldEval, source) -> fieldEval)
+        Map.entry(DataType.COUNTER_INTEGER, (fieldEval, source) -> fieldEval)
     );
 
     @FunctionInfo(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java
index 8d00b19f60657..182c1c923a96d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -22,15 +21,15 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeDoubleToLong;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToLong;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToLong;
 
@@ -45,8 +44,8 @@ public class ToLong extends AbstractConvertFunction {
         Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new),
         Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new),
         Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new), // CastIntToLongEvaluator would be a candidate, but not MV'd
-        Map.entry(DataTypes.COUNTER_LONG, (field, source) -> field),
-        Map.entry(DataTypes.COUNTER_INTEGER, ToLongFromIntEvaluator.Factory::new)
+        Map.entry(DataType.COUNTER_LONG, (field, source) -> field),
+        Map.entry(DataType.COUNTER_INTEGER, ToLongFromIntEvaluator.Factory::new)
     );
 
     @FunctionInfo(
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadians.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadians.java
index 830463eca38ed..2fd8ae1eb0bdd 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadians.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadians.java
@@ -20,10 +20,10 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 
 /**
  * Converts from <a href="https://en.wikipedia.org/wiki/Degree_(angle)">degrees</a>
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
index 04318ba4fff64..141fa067e54e1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
@@ -21,20 +21,20 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.numericBooleanToString;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java
index b9206c592010d..ad6e935f7d6a7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java
@@ -21,14 +21,14 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.booleanToUnsignedLong;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.doubleToUnsignedLong;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.intToUnsignedLong;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java
index 94c48bace1f34..1941dd00837b5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java
@@ -20,9 +20,9 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToVersion;
 
 public class ToVersion extends AbstractConvertFunction {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/BinaryDateTimeFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/BinaryDateTimeFunction.java
index ce949c9e6b390..74f0dae76c425 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/BinaryDateTimeFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/BinaryDateTimeFunction.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.time.ZoneId;
 import java.time.ZoneOffset;
@@ -30,7 +29,7 @@ protected BinaryDateTimeFunction(Source source, Expression argument, Expression
 
     @Override
     public DataType dataType() {
-        return DataTypes.DATETIME;
+        return DataType.DATETIME;
     }
 
     public Expression timestampField() {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java
index 33db8430f20bd..42e20a9a4615e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -219,7 +218,7 @@ public boolean foldable() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.INTEGER;
+        return DataType.INTEGER;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java
index 11759cdab1584..c28c5e417c152 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -123,7 +122,7 @@ protected NodeInfo<? extends Expression> info() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.LONG;
+        return DataType.LONG;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java
index 6690d495b0ecb..bcc5d7cb16050 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -62,7 +61,7 @@ Date format (optional).  If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.S
 
     @Override
     public DataType dataType() {
-        return DataTypes.KEYWORD;
+        return DataType.KEYWORD;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java
index 3852d57772e7d..d68664afe8418 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -67,7 +66,7 @@ public DateParse(
 
     @Override
     public DataType dataType() {
-        return DataTypes.DATETIME;
+        return DataType.DATETIME;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
index 49f55e42931ac..ddd51d281105d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -82,7 +81,7 @@ protected TypeResolution resolveType() {
     }
 
     public DataType dataType() {
-        return DataTypes.DATETIME;
+        return DataType.DATETIME;
     }
 
     @Evaluator
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java
index 88e9d2b60b5b5..fe54cfd186fec 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction;
@@ -60,7 +59,7 @@ public boolean foldable() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DATETIME;
+        return DataType.DATETIME;
     }
 
     @Evaluator
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java
index 12d2692460cb3..e2c2395446ed6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -105,7 +104,7 @@ static boolean process(BytesRef ip, BytesRef[] cidrs) {
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
index 9d429a620ed6e..d00d1b2c35fcb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -35,7 +34,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isIPAndExact;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 
 /**
  * Truncates an IP value to a given prefix length.
@@ -172,7 +171,7 @@ private static void makePrefix(BytesRef ip, BytesRef scratch, int fullBytes, int
 
     @Override
     public DataType dataType() {
-        return DataTypes.IP;
+        return DataType.IP;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java
index 3898b342f007e..b821b8449a5a0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -57,16 +57,16 @@ static int process(int fieldVal) {
     @Override
     public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
         var field = toEvaluator.apply(field());
-        if (dataType() == DataTypes.DOUBLE) {
+        if (dataType() == DataType.DOUBLE) {
             return new AbsDoubleEvaluator.Factory(source(), field);
         }
-        if (dataType() == DataTypes.UNSIGNED_LONG) {
+        if (dataType() == DataType.UNSIGNED_LONG) {
             return field;
         }
-        if (dataType() == DataTypes.LONG) {
+        if (dataType() == DataType.LONG) {
             return new AbsLongEvaluator.Factory(source(), field);
         }
-        if (dataType() == DataTypes.INTEGER) {
+        if (dataType() == DataType.INTEGER) {
             return new AbsIntEvaluator.Factory(source(), field);
         }
         throw EsqlIllegalArgumentException.illegalDataType(dataType());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java
index 1643d1b21ca5d..fee52567d161e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction;
 
 import java.util.function.Function;
@@ -35,7 +34,7 @@ abstract class AbstractTrigonometricFunction extends UnaryScalarFunction {
 
     @Override
     public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
-        return doubleEvaluator(Cast.cast(source(), field().dataType(), DataTypes.DOUBLE, toEvaluator.apply(field())));
+        return doubleEvaluator(Cast.cast(source(), field().dataType(), DataType.DOUBLE, toEvaluator.apply(field())));
     }
 
     @Override
@@ -49,6 +48,6 @@ protected final TypeResolution resolveType() {
 
     @Override
     public final DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java
index 50b1e6fc0b17e..a2af991a244c3 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -74,7 +73,7 @@ static double process(double y, double x) {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
@@ -97,8 +96,8 @@ public boolean foldable() {
 
     @Override
     public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
-        var yEval = Cast.cast(source(), y.dataType(), DataTypes.DOUBLE, toEvaluator.apply(y));
-        var xEval = Cast.cast(source(), x.dataType(), DataTypes.DOUBLE, toEvaluator.apply(x));
+        var yEval = Cast.cast(source(), y.dataType(), DataType.DOUBLE, toEvaluator.apply(y));
+        var xEval = Cast.cast(source(), x.dataType(), DataType.DOUBLE, toEvaluator.apply(x));
         return new Atan2Evaluator.Factory(source(), yEval, xEval);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cast.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cast.java
index 5ffac6fb2b015..f4936f8ee37c6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cast.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cast.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.intToUnsignedLong;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.longToUnsignedLong;
@@ -27,31 +26,31 @@ public static ExpressionEvaluator.Factory cast(Source source, DataType current,
         if (current == required) {
             return in;
         }
-        if (current == DataTypes.NULL || required == DataTypes.NULL) {
+        if (current == DataType.NULL || required == DataType.NULL) {
             return EvalOperator.CONSTANT_NULL_FACTORY;
         }
-        if (required == DataTypes.DOUBLE) {
-            if (current == DataTypes.LONG) {
+        if (required == DataType.DOUBLE) {
+            if (current == DataType.LONG) {
                 return new CastLongToDoubleEvaluator.Factory(source, in);
             }
-            if (current == DataTypes.INTEGER) {
+            if (current == DataType.INTEGER) {
                 return new CastIntToDoubleEvaluator.Factory(source, in);
             }
-            if (current == DataTypes.UNSIGNED_LONG) {
+            if (current == DataType.UNSIGNED_LONG) {
                 return new CastUnsignedLongToDoubleEvaluator.Factory(source, in);
             }
             throw cantCast(current, required);
         }
-        if (required == DataTypes.UNSIGNED_LONG) {
-            if (current == DataTypes.LONG) {
+        if (required == DataType.UNSIGNED_LONG) {
+            if (current == DataType.LONG) {
                 return new CastLongToUnsignedLongEvaluator.Factory(source, in);
             }
-            if (current == DataTypes.INTEGER) {
+            if (current == DataType.INTEGER) {
                 return new CastIntToUnsignedLongEvaluator.Factory(source, in);
             }
         }
-        if (required == DataTypes.LONG) {
-            if (current == DataTypes.INTEGER) {
+        if (required == DataType.LONG) {
+            if (current == DataType.INTEGER) {
                 return new CastIntToLongEvaluator.Factory(source, in);
             }
             throw cantCast(current, required);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java
index 8eec5ff9b7aa1..43c0353de490a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -47,16 +46,16 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
         var field = toEvaluator.apply(field());
         var fieldType = field().dataType();
 
-        if (fieldType == DataTypes.DOUBLE) {
+        if (fieldType == DataType.DOUBLE) {
             return new CbrtDoubleEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.INTEGER) {
+        if (fieldType == DataType.INTEGER) {
             return new CbrtIntEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.LONG) {
+        if (fieldType == DataType.LONG) {
             return new CbrtLongEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.UNSIGNED_LONG) {
+        if (fieldType == DataType.UNSIGNED_LONG) {
             return new CbrtUnsignedLongEvaluator.Factory(source(), field);
         }
 
@@ -95,7 +94,7 @@ protected NodeInfo<? extends Expression> info() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/DoubleConstantFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/DoubleConstantFunction.java
index 6f9ca89796f0d..8c42fb22db0ba 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/DoubleConstantFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/DoubleConstantFunction.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 /**
  * Function that emits constants, like Euler's number.
@@ -29,7 +28,7 @@ public final boolean foldable() {
 
     @Override
     public final DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java
index f60b0341dbb18..97007f10b31bc 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -115,14 +114,14 @@ protected NodeInfo<? extends Expression> info() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
     public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
-        var valueEval = Cast.cast(source(), value.dataType(), DataTypes.DOUBLE, toEvaluator.apply(value));
+        var valueEval = Cast.cast(source(), value.dataType(), DataType.DOUBLE, toEvaluator.apply(value));
         if (base != null) {
-            var baseEval = Cast.cast(source(), base.dataType(), DataTypes.DOUBLE, toEvaluator.apply(base));
+            var baseEval = Cast.cast(source(), base.dataType(), DataType.DOUBLE, toEvaluator.apply(base));
             return new LogEvaluator.Factory(source(), baseEval, valueEval);
         }
         return new LogConstantEvaluator.Factory(source(), valueEval);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java
index ad08b34e3c53a..69b7efac9b7e0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -53,16 +52,16 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
         var field = toEvaluator.apply(field());
         var fieldType = field().dataType();
 
-        if (fieldType == DataTypes.DOUBLE) {
+        if (fieldType == DataType.DOUBLE) {
             return new Log10DoubleEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.INTEGER) {
+        if (fieldType == DataType.INTEGER) {
             return new Log10IntEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.LONG) {
+        if (fieldType == DataType.LONG) {
             return new Log10LongEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.UNSIGNED_LONG) {
+        if (fieldType == DataType.UNSIGNED_LONG) {
             return new Log10UnsignedLongEvaluator.Factory(source(), field);
         }
 
@@ -113,7 +112,7 @@ protected NodeInfo<? extends Expression> info() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java
index 0e60e25af1981..ab5282e665ebf 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -103,13 +102,13 @@ public Expression exponent() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
     public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
-        var baseEval = Cast.cast(source(), base.dataType(), DataTypes.DOUBLE, toEvaluator.apply(base));
-        var expEval = Cast.cast(source(), exponent.dataType(), DataTypes.DOUBLE, toEvaluator.apply(exponent));
+        var baseEval = Cast.cast(source(), base.dataType(), DataType.DOUBLE, toEvaluator.apply(base));
+        var expEval = Cast.cast(source(), exponent.dataType(), DataType.DOUBLE, toEvaluator.apply(exponent));
         return new PowEvaluator.Factory(source(), baseEval, expEval);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java
index d1f6eba4081ba..0d7ca026c81ad 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -144,16 +143,16 @@ public DataType dataType() {
     @Override
     public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
         DataType fieldType = dataType();
-        if (fieldType == DataTypes.DOUBLE) {
+        if (fieldType == DataType.DOUBLE) {
             return toEvaluator(toEvaluator, RoundDoubleNoDecimalsEvaluator.Factory::new, RoundDoubleEvaluator.Factory::new);
         }
-        if (fieldType == DataTypes.INTEGER) {
+        if (fieldType == DataType.INTEGER) {
             return toEvaluator(toEvaluator, EVALUATOR_IDENTITY, RoundIntEvaluator.Factory::new);
         }
-        if (fieldType == DataTypes.LONG) {
+        if (fieldType == DataType.LONG) {
             return toEvaluator(toEvaluator, EVALUATOR_IDENTITY, RoundLongEvaluator.Factory::new);
         }
-        if (fieldType == DataTypes.UNSIGNED_LONG) {
+        if (fieldType == DataType.UNSIGNED_LONG) {
             return toEvaluator(toEvaluator, EVALUATOR_IDENTITY, RoundUnsignedLongEvaluator.Factory::new);
         }
         throw EsqlIllegalArgumentException.illegalDataType(fieldType);
@@ -168,7 +167,7 @@ private ExpressionEvaluator.Factory toEvaluator(
         if (decimals == null) {
             return noDecimals.apply(source(), fieldEvaluator);
         }
-        var decimalsEvaluator = Cast.cast(source(), decimals().dataType(), DataTypes.LONG, toEvaluator.apply(decimals()));
+        var decimalsEvaluator = Cast.cast(source(), decimals().dataType(), DataType.LONG, toEvaluator.apply(decimals()));
         return withDecimals.apply(source(), fieldEvaluator, decimalsEvaluator);
     }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java
index 84126e0d10797..7769e8c6c4a2e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -49,16 +48,16 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(
         var field = toEvaluator.apply(field());
         var fieldType = field().dataType();
 
-        if (fieldType == DataTypes.DOUBLE) {
+        if (fieldType == DataType.DOUBLE) {
             return new SignumDoubleEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.INTEGER) {
+        if (fieldType == DataType.INTEGER) {
             return new SignumIntEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.LONG) {
+        if (fieldType == DataType.LONG) {
             return new SignumLongEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.UNSIGNED_LONG) {
+        if (fieldType == DataType.UNSIGNED_LONG) {
             return new SignumUnsignedLongEvaluator.Factory(source(), field);
         }
 
@@ -77,7 +76,7 @@ protected NodeInfo<? extends Expression> info() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Evaluator(extraName = "Double")
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java
index 65e24bb5de13e..a27929b0b5d0a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -47,16 +46,16 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
         var field = toEvaluator.apply(field());
         var fieldType = field().dataType();
 
-        if (fieldType == DataTypes.DOUBLE) {
+        if (fieldType == DataType.DOUBLE) {
             return new SqrtDoubleEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.INTEGER) {
+        if (fieldType == DataType.INTEGER) {
             return new SqrtIntEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.LONG) {
+        if (fieldType == DataType.LONG) {
             return new SqrtLongEvaluator.Factory(source(), field);
         }
-        if (fieldType == DataTypes.UNSIGNED_LONG) {
+        if (fieldType == DataType.UNSIGNED_LONG) {
             return new SqrtUnsignedLongEvaluator.Factory(source(), field);
         }
 
@@ -104,7 +103,7 @@ protected NodeInfo<? extends Expression> info() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java
index ba217c6fa0392..787bf3e5efd1c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -55,7 +54,7 @@ protected TypeResolution resolveFieldType() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
@@ -63,7 +62,7 @@ protected ExpressionEvaluator.Factory evaluator(ExpressionEvaluator.Factory fiel
         return switch (PlannerUtils.toElementType(field().dataType())) {
             case DOUBLE -> new MvAvgDoubleEvaluator.Factory(fieldEval);
             case INT -> new MvAvgIntEvaluator.Factory(fieldEval);
-            case LONG -> field().dataType() == DataTypes.UNSIGNED_LONG
+            case LONG -> field().dataType() == DataType.UNSIGNED_LONG
                 ? new MvAvgUnsignedLongEvaluator.Factory(fieldEval)
                 : new MvAvgLongEvaluator.Factory(fieldEval);
             case NULL -> EvalOperator.CONSTANT_NULL_FACTORY;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java
index dd4b552999a00..3e37a739147cf 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -70,7 +69,7 @@ protected TypeResolution resolveType() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.KEYWORD;
+        return DataType.KEYWORD;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java
index 26c3345240cd2..b2afef4f2235e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -66,7 +65,7 @@ protected TypeResolution resolveFieldType() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.INTEGER;
+        return DataType.INTEGER;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java
index f936cd6fdfd1a..8d3177926f2e6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -70,7 +70,7 @@ protected ExpressionEvaluator.Factory evaluator(ExpressionEvaluator.Factory fiel
         return switch (PlannerUtils.toElementType(field().dataType())) {
             case DOUBLE -> new MvMedianDoubleEvaluator.Factory(fieldEval);
             case INT -> new MvMedianIntEvaluator.Factory(fieldEval);
-            case LONG -> field().dataType() == DataTypes.UNSIGNED_LONG
+            case LONG -> field().dataType() == DataType.UNSIGNED_LONG
                 ? new MvMedianUnsignedLongEvaluator.Factory(fieldEval)
                 : new MvMedianLongEvaluator.Factory(fieldEval);
             default -> throw EsqlIllegalArgumentException.illegalDataType(field.dataType());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
index d066745cc65b9..40e9f90df9dc6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java
@@ -39,7 +39,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToInt;
 
 /**
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
index 3e1d7aabe7d39..744491b30f702 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -57,7 +56,7 @@
 public class MvSort extends EsqlScalarFunction implements OptionalArgument, Validatable {
     private final Expression field, order;
 
-    private static final Literal ASC = new Literal(Source.EMPTY, "ASC", DataTypes.KEYWORD);
+    private static final Literal ASC = new Literal(Source.EMPTY, "ASC", DataType.KEYWORD);
 
     @FunctionInfo(
         returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" },
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java
index c3cee1a227325..e14bc401a058a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -57,7 +57,7 @@ protected ExpressionEvaluator.Factory evaluator(ExpressionEvaluator.Factory fiel
         return switch (PlannerUtils.toElementType(field().dataType())) {
             case DOUBLE -> new MvSumDoubleEvaluator.Factory(fieldEval);
             case INT -> new MvSumIntEvaluator.Factory(source(), fieldEval);
-            case LONG -> field().dataType() == DataTypes.UNSIGNED_LONG
+            case LONG -> field().dataType() == DataType.UNSIGNED_LONG
                 ? new MvSumUnsignedLongEvaluator.Factory(source(), fieldEval)
                 : new MvSumLongEvaluator.Factory(source(), fieldEval);
             case NULL -> EvalOperator.CONSTANT_NULL_FACTORY;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java
index 8ca09569015a4..6298057b16013 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -39,7 +38,7 @@
  */
 public class MvZip extends EsqlScalarFunction implements OptionalArgument, EvaluatorMapper {
     private final Expression mvLeft, mvRight, delim;
-    private static final Literal COMMA = new Literal(Source.EMPTY, ",", DataTypes.TEXT);
+    private static final Literal COMMA = new Literal(Source.EMPTY, ",", DataType.TEXT);
 
     @FunctionInfo(
         returnType = { "keyword" },
@@ -113,7 +112,7 @@ protected NodeInfo<? extends Expression> info() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.KEYWORD;
+        return DataType.KEYWORD;
     }
 
     private static void buildOneSide(BytesRefBlock.Builder builder, int start, int end, BytesRefBlock field, BytesRef fieldScratch) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java
index 5c823f47f794f..a367e2f75b7e9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java
@@ -34,7 +34,7 @@
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NULL;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
 
 /**
  * Function returning the first non-null value.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
index 729262d4d5870..63fdb9b5bc774 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java
@@ -36,10 +36,10 @@
 import java.util.Map;
 import java.util.Set;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2Ds;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.makeGeometryFromLiteral;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
index f66533ba38162..26d48831fdd81 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java
@@ -34,10 +34,10 @@
 import java.util.Map;
 import java.util.Set;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
index 8abc678415e71..c0794f59dcf81 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java
@@ -34,10 +34,10 @@
 import java.util.Map;
 import java.util.Set;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
index d2527c5a8bcd2..064df31e35cb2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -40,9 +39,9 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isNull;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isNull;
 import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatial;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D;
@@ -65,7 +64,7 @@ protected SpatialRelatesFunction(Source source, Expression left, Expression righ
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
index 8ca7a40639e1f..6568fd42d44c7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java
@@ -35,10 +35,10 @@
 import java.util.Map;
 import java.util.Set;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader;
 import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java
index b2ece9388e35c..2af1a353a39dc 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java
@@ -23,7 +23,7 @@
 import java.util.List;
 import java.util.function.Function;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED;
 import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatialPoint;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java
index b52d6929ec3a7..266c5f986c526 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java
@@ -23,7 +23,7 @@
 import java.util.List;
 import java.util.function.Function;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED;
 import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isSpatialPoint;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java
index 670c96d880f21..d01edbe7024e8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -54,7 +53,7 @@ public Concat(
 
     @Override
     public DataType dataType() {
-        return DataTypes.KEYWORD;
+        return DataType.KEYWORD;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java
index 11a33d9a038e1..767563ed4112a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -58,7 +57,7 @@ public EndsWith(
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java
index 422ae9532aab7..384874e173658 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -30,7 +29,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 
 /**
  * {code left(foo, len)} is an alias to {code substring(foo, 0, len)}
@@ -95,7 +94,7 @@ protected NodeInfo<? extends Expression> info() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.KEYWORD;
+        return DataType.KEYWORD;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java
index cad8bf2770529..e2beda9612b04 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -47,7 +46,7 @@ public Length(
 
     @Override
     public DataType dataType() {
-        return DataTypes.INTEGER;
+        return DataType.INTEGER;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java
index 067a16cc3fc66..1669a64ec83d2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -64,7 +63,7 @@ public Locate(
 
     @Override
     public DataType dataType() {
-        return DataTypes.INTEGER;
+        return DataType.INTEGER;
     }
 
     @Override
@@ -82,7 +81,7 @@ protected TypeResolution resolveType() {
             return resolution;
         }
 
-        return start == null ? TypeResolution.TYPE_RESOLVED : isType(start, dt -> dt == DataTypes.INTEGER, sourceText(), THIRD, "integer");
+        return start == null ? TypeResolution.TYPE_RESOLVED : isType(start, dt -> dt == DataType.INTEGER, sourceText(), THIRD, "integer");
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java
index c0871974bcc1a..7318c50a2e54d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -63,7 +62,7 @@ public Replace(
 
     @Override
     public DataType dataType() {
-        return DataTypes.KEYWORD;
+        return DataType.KEYWORD;
     }
 
     @Override
@@ -125,7 +124,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
         var strEval = toEvaluator.apply(str);
         var newStrEval = toEvaluator.apply(newStr);
 
-        if (regex.foldable() && regex.dataType() == DataTypes.KEYWORD) {
+        if (regex.foldable() && regex.dataType() == DataType.KEYWORD) {
             Pattern regexPattern;
             try {
                 regexPattern = Pattern.compile(((BytesRef) regex.fold()).utf8ToString());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java
index cc051d93b6cbc..893e5ebb6a8cb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -30,7 +29,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 
 /**
  * {code right(foo, len)} is an alias to {code substring(foo, foo.length-len, len)}
@@ -99,7 +98,7 @@ protected NodeInfo<? extends Expression> info() {
 
     @Override
     public DataType dataType() {
-        return DataTypes.KEYWORD;
+        return DataType.KEYWORD;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java
index edb052821bef3..99c109d72ae63 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -57,7 +56,7 @@ public Split(
 
     @Override
     public DataType dataType() {
-        return DataTypes.KEYWORD;
+        return DataType.KEYWORD;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java
index 13a9fb42d2db1..f1d67f317a60f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -58,7 +57,7 @@ public StartsWith(
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java
index 4f3d0e15eef72..94b9f06b63b5d 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -31,7 +30,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 
 public class Substring extends EsqlScalarFunction implements OptionalArgument {
 
@@ -72,7 +71,7 @@ public Substring(
 
     @Override
     public DataType dataType() {
-        return DataTypes.KEYWORD;
+        return DataType.KEYWORD;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
index 38f44988d6ac6..2f2af5c0ff8d7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.time.Duration;
@@ -22,11 +21,11 @@
 import java.util.Collection;
 import java.util.function.Function;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isDateTime;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isNull;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isNull;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isDateTimeOrTemporal;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
 
@@ -57,7 +56,7 @@ interface DatetimeArithmeticEvaluator {
     protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) {
         return TypeResolutions.isType(
             e,
-            t -> t.isNumeric() || EsqlDataTypes.isDateTimeOrTemporal(t) || DataTypes.isNull(t),
+            t -> t.isNumeric() || EsqlDataTypes.isDateTimeOrTemporal(t) || DataType.isNull(t),
             sourceText(),
             paramOrdinal,
             "datetime",
@@ -91,7 +90,7 @@ protected TypeResolution checkCompatibility() {
     }
 
     /**
-     * Override this to allow processing literals of type {@link DataTypes#DATE_PERIOD} when folding constants.
+     * Override this to allow processing literals of type {@link DataType#DATE_PERIOD} when folding constants.
      * Used in {@link DateTimeArithmeticOperation#fold()}.
      * @param left the left period
      * @param right the right period
@@ -100,7 +99,7 @@ protected TypeResolution checkCompatibility() {
     abstract Period fold(Period left, Period right);
 
     /**
-     * Override this to allow processing literals of type {@link DataTypes#TIME_DURATION} when folding constants.
+     * Override this to allow processing literals of type {@link DataType#TIME_DURATION} when folding constants.
      * Used in {@link DateTimeArithmeticOperation#fold()}.
      * @param left the left duration
      * @param right the right duration
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java
index f43b217d7ebef..6d63551abd314 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryArithmeticOperation;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry;
@@ -24,10 +23,10 @@
 import java.util.function.Function;
 
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 
 public abstract class EsqlArithmeticOperation extends ArithmeticOperation implements EvaluatorMapper {
 
@@ -131,8 +130,8 @@ protected TypeResolution checkCompatibility() {
         // This checks that unsigned longs should only be compatible with other unsigned longs
         DataType leftType = left().dataType();
         DataType rightType = right().dataType();
-        if ((rightType == UNSIGNED_LONG && (false == (leftType == UNSIGNED_LONG || leftType == DataTypes.NULL)))
-            || (leftType == UNSIGNED_LONG && (false == (rightType == UNSIGNED_LONG || rightType == DataTypes.NULL)))) {
+        if ((rightType == UNSIGNED_LONG && (false == (leftType == UNSIGNED_LONG || leftType == DataType.NULL)))
+            || (leftType == UNSIGNED_LONG && (false == (rightType == UNSIGNED_LONG || rightType == DataType.NULL)))) {
             return new TypeResolution(formatIncompatibleTypesMessage(symbol(), leftType, rightType));
         }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
index 0847c740805d9..50fb5c58e2005 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.Warnings;
 import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction;
 
@@ -26,8 +25,8 @@
 
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
 
 public class Neg extends UnaryScalarFunction {
@@ -47,13 +46,13 @@ public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEv
             var f = toEvaluator.apply(field());
             ExpressionEvaluator.Factory factory = null;
 
-            if (type == DataTypes.INTEGER) {
+            if (type == DataType.INTEGER) {
                 factory = new NegIntsEvaluator.Factory(source(), f);
             }
             // Unsigned longs are unsupported by choice; negating them would require implicitly converting to long.
-            else if (type == DataTypes.LONG) {
+            else if (type == DataType.LONG) {
                 factory = new NegLongsEvaluator.Factory(source(), f);
-            } else if (type == DataTypes.DOUBLE) {
+            } else if (type == DataType.DOUBLE) {
                 factory = new NegDoublesEvaluator.Factory(source(), f);
             }
 
@@ -98,7 +97,7 @@ public final Object fold() {
     protected TypeResolution resolveType() {
         return isType(
             field(),
-            dt -> dt != DataTypes.UNSIGNED_LONG && (dt.isNumeric() || isTemporalAmount(dt)),
+            dt -> dt != DataType.UNSIGNED_LONG && (dt.isNumeric() || isTemporalAmount(dt)),
             sourceText(),
             DEFAULT,
             "numeric",
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java
index 7c1ece4efaa12..b2ae8cff6a697 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.time.DateTimeException;
@@ -47,7 +47,7 @@ public Sub(Source source, Expression left, Expression right) {
     protected TypeResolution resolveType() {
         TypeResolution resolution = super.resolveType();
         // As opposed to general date time arithmetics, we cannot subtract a datetime from something else.
-        if (resolution.resolved() && EsqlDataTypes.isDateTimeOrTemporal(dataType()) && DataTypes.isDateTime(right().dataType())) {
+        if (resolution.resolved() && EsqlDataTypes.isDateTimeOrTemporal(dataType()) && DataType.isDateTime(right().dataType())) {
             return new TypeResolution(
                 format(
                     null,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
index 44c7cb39b9a1a..e73cf91cd52a8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -21,20 +20,20 @@
 
 public class Equals extends EsqlBinaryComparison implements Negatable<EsqlBinaryComparison> {
     private static final Map<DataType, EsqlArithmeticOperation.BinaryEvaluator> evaluatorMap = Map.ofEntries(
-        Map.entry(DataTypes.BOOLEAN, EqualsBoolsEvaluator.Factory::new),
-        Map.entry(DataTypes.INTEGER, EqualsIntsEvaluator.Factory::new),
-        Map.entry(DataTypes.DOUBLE, EqualsDoublesEvaluator.Factory::new),
-        Map.entry(DataTypes.LONG, EqualsLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.UNSIGNED_LONG, EqualsLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.DATETIME, EqualsLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.GEO_POINT, EqualsGeometriesEvaluator.Factory::new),
-        Map.entry(DataTypes.CARTESIAN_POINT, EqualsGeometriesEvaluator.Factory::new),
-        Map.entry(DataTypes.GEO_SHAPE, EqualsGeometriesEvaluator.Factory::new),
-        Map.entry(DataTypes.CARTESIAN_SHAPE, EqualsGeometriesEvaluator.Factory::new),
-        Map.entry(DataTypes.KEYWORD, EqualsKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.TEXT, EqualsKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.VERSION, EqualsKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.IP, EqualsKeywordsEvaluator.Factory::new)
+        Map.entry(DataType.BOOLEAN, EqualsBoolsEvaluator.Factory::new),
+        Map.entry(DataType.INTEGER, EqualsIntsEvaluator.Factory::new),
+        Map.entry(DataType.DOUBLE, EqualsDoublesEvaluator.Factory::new),
+        Map.entry(DataType.LONG, EqualsLongsEvaluator.Factory::new),
+        Map.entry(DataType.UNSIGNED_LONG, EqualsLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATETIME, EqualsLongsEvaluator.Factory::new),
+        Map.entry(DataType.GEO_POINT, EqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataType.CARTESIAN_POINT, EqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataType.GEO_SHAPE, EqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataType.CARTESIAN_SHAPE, EqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataType.KEYWORD, EqualsKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.TEXT, EqualsKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.VERSION, EqualsKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.IP, EqualsKeywordsEvaluator.Factory::new)
     );
 
     public Equals(Source source, Expression left, Expression right) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
index a1a1bf6b2c19a..41dafecbff76e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparisonProcessor;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
@@ -30,7 +29,7 @@
 import java.util.function.Function;
 
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 
 public abstract class EsqlBinaryComparison extends BinaryComparison implements EvaluatorMapper {
 
@@ -182,16 +181,16 @@ protected TypeResolution checkCompatibility() {
         DataType rightType = right().dataType();
 
         // Unsigned long is only interoperable with other unsigned longs
-        if ((rightType == UNSIGNED_LONG && (false == (leftType == UNSIGNED_LONG || leftType == DataTypes.NULL)))
-            || (leftType == UNSIGNED_LONG && (false == (rightType == UNSIGNED_LONG || rightType == DataTypes.NULL)))) {
+        if ((rightType == UNSIGNED_LONG && (false == (leftType == UNSIGNED_LONG || leftType == DataType.NULL)))
+            || (leftType == UNSIGNED_LONG && (false == (rightType == UNSIGNED_LONG || rightType == DataType.NULL)))) {
             return new TypeResolution(formatIncompatibleTypesMessage());
         }
 
         if ((leftType.isNumeric() && rightType.isNumeric())
-            || (DataTypes.isString(leftType) && DataTypes.isString(rightType))
+            || (DataType.isString(leftType) && DataType.isString(rightType))
             || leftType.equals(rightType)
-            || DataTypes.isNull(leftType)
-            || DataTypes.isNull(rightType)) {
+            || DataType.isNull(leftType)
+            || DataType.isNull(rightType)) {
             return TypeResolution.TYPE_RESOLVED;
         }
         return new TypeResolution(formatIncompatibleTypesMessage());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java
index 655641be88f84..da639b328b7c2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -21,15 +20,15 @@
 
 public class GreaterThan extends EsqlBinaryComparison implements Negatable<EsqlBinaryComparison> {
     private static final Map<DataType, EsqlArithmeticOperation.BinaryEvaluator> evaluatorMap = Map.ofEntries(
-        Map.entry(DataTypes.INTEGER, GreaterThanIntsEvaluator.Factory::new),
-        Map.entry(DataTypes.DOUBLE, GreaterThanDoublesEvaluator.Factory::new),
-        Map.entry(DataTypes.LONG, GreaterThanLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.UNSIGNED_LONG, GreaterThanLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.DATETIME, GreaterThanLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.KEYWORD, GreaterThanKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.TEXT, GreaterThanKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.VERSION, GreaterThanKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.IP, GreaterThanKeywordsEvaluator.Factory::new)
+        Map.entry(DataType.INTEGER, GreaterThanIntsEvaluator.Factory::new),
+        Map.entry(DataType.DOUBLE, GreaterThanDoublesEvaluator.Factory::new),
+        Map.entry(DataType.LONG, GreaterThanLongsEvaluator.Factory::new),
+        Map.entry(DataType.UNSIGNED_LONG, GreaterThanLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATETIME, GreaterThanLongsEvaluator.Factory::new),
+        Map.entry(DataType.KEYWORD, GreaterThanKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.TEXT, GreaterThanKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.VERSION, GreaterThanKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.IP, GreaterThanKeywordsEvaluator.Factory::new)
     );
 
     public GreaterThan(Source source, Expression left, Expression right) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java
index 48ed5f513bbdc..0644cd5df9038 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -21,15 +20,15 @@
 
 public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatable<EsqlBinaryComparison> {
     private static final Map<DataType, EsqlArithmeticOperation.BinaryEvaluator> evaluatorMap = Map.ofEntries(
-        Map.entry(DataTypes.INTEGER, GreaterThanOrEqualIntsEvaluator.Factory::new),
-        Map.entry(DataTypes.DOUBLE, GreaterThanOrEqualDoublesEvaluator.Factory::new),
-        Map.entry(DataTypes.LONG, GreaterThanOrEqualLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.UNSIGNED_LONG, GreaterThanOrEqualLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.DATETIME, GreaterThanOrEqualLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.KEYWORD, GreaterThanOrEqualKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.TEXT, GreaterThanOrEqualKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.VERSION, GreaterThanOrEqualKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.IP, GreaterThanOrEqualKeywordsEvaluator.Factory::new)
+        Map.entry(DataType.INTEGER, GreaterThanOrEqualIntsEvaluator.Factory::new),
+        Map.entry(DataType.DOUBLE, GreaterThanOrEqualDoublesEvaluator.Factory::new),
+        Map.entry(DataType.LONG, GreaterThanOrEqualLongsEvaluator.Factory::new),
+        Map.entry(DataType.UNSIGNED_LONG, GreaterThanOrEqualLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATETIME, GreaterThanOrEqualLongsEvaluator.Factory::new),
+        Map.entry(DataType.KEYWORD, GreaterThanOrEqualKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.TEXT, GreaterThanOrEqualKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.VERSION, GreaterThanOrEqualKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.IP, GreaterThanOrEqualKeywordsEvaluator.Factory::new)
     );
 
     public GreaterThanOrEqual(Source source, Expression left, Expression right) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
index c46b3588df3df..17fca1e1cff88 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions;
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
@@ -66,7 +65,7 @@ public Boolean fold() {
 
     @Override
     protected boolean areCompatible(DataType left, DataType right) {
-        if (left == DataTypes.UNSIGNED_LONG || right == DataTypes.UNSIGNED_LONG) {
+        if (left == DataType.UNSIGNED_LONG || right == DataType.UNSIGNED_LONG) {
             // automatic numerical conversions not applicable for UNSIGNED_LONG, see Verifier#validateUnsignedLongOperator().
             return left == right;
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java
index 6275db7d78524..9302f6e9c5a77 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 public abstract class InsensitiveBinaryComparison extends BinaryScalarFunction {
 
@@ -20,7 +19,7 @@ protected InsensitiveBinaryComparison(Source source, Expression left, Expression
 
     @Override
     public DataType dataType() {
-        return DataTypes.BOOLEAN;
+        return DataType.BOOLEAN;
     }
 
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java
index f73960fc73e88..0afc9e0280f4c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.evaluator.mapper.ExpressionMapper;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast;
 import org.elasticsearch.xpack.esql.planner.Layout;
@@ -36,7 +35,7 @@ public final ExpressionEvaluator.Factory map(InsensitiveEquals bc, Layout layout
 
         var leftEval = toEvaluator(bc.left(), layout);
         var rightEval = toEvaluator(bc.right(), layout);
-        if (leftType == DataTypes.KEYWORD || leftType == DataTypes.TEXT) {
+        if (leftType == DataType.KEYWORD || leftType == DataType.TEXT) {
             if (bc.right().foldable() && EsqlDataTypes.isString(rightType)) {
                 BytesRef rightVal = BytesRefs.toBytesRef(bc.right().fold());
                 Automaton automaton = InsensitiveEquals.automaton(rightVal);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java
index c1e8d167c9d2a..8c6824a9827d0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -22,15 +21,15 @@
 public class LessThan extends EsqlBinaryComparison implements Negatable<EsqlBinaryComparison> {
 
     private static final Map<DataType, EsqlArithmeticOperation.BinaryEvaluator> evaluatorMap = Map.ofEntries(
-        Map.entry(DataTypes.INTEGER, LessThanIntsEvaluator.Factory::new),
-        Map.entry(DataTypes.DOUBLE, LessThanDoublesEvaluator.Factory::new),
-        Map.entry(DataTypes.LONG, LessThanLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.UNSIGNED_LONG, LessThanLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.DATETIME, LessThanLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.KEYWORD, LessThanKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.TEXT, LessThanKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.VERSION, LessThanKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.IP, LessThanKeywordsEvaluator.Factory::new)
+        Map.entry(DataType.INTEGER, LessThanIntsEvaluator.Factory::new),
+        Map.entry(DataType.DOUBLE, LessThanDoublesEvaluator.Factory::new),
+        Map.entry(DataType.LONG, LessThanLongsEvaluator.Factory::new),
+        Map.entry(DataType.UNSIGNED_LONG, LessThanLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATETIME, LessThanLongsEvaluator.Factory::new),
+        Map.entry(DataType.KEYWORD, LessThanKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.TEXT, LessThanKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.VERSION, LessThanKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.IP, LessThanKeywordsEvaluator.Factory::new)
     );
 
     public LessThan(Source source, Expression left, Expression right) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java
index a2daa8b48ea63..0a18c44ea2891 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -21,15 +20,15 @@
 
 public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable<EsqlBinaryComparison> {
     private static final Map<DataType, EsqlArithmeticOperation.BinaryEvaluator> evaluatorMap = Map.ofEntries(
-        Map.entry(DataTypes.INTEGER, LessThanOrEqualIntsEvaluator.Factory::new),
-        Map.entry(DataTypes.DOUBLE, LessThanOrEqualDoublesEvaluator.Factory::new),
-        Map.entry(DataTypes.LONG, LessThanOrEqualLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.UNSIGNED_LONG, LessThanOrEqualLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.DATETIME, LessThanOrEqualLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.KEYWORD, LessThanOrEqualKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.TEXT, LessThanOrEqualKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.VERSION, LessThanOrEqualKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.IP, LessThanOrEqualKeywordsEvaluator.Factory::new)
+        Map.entry(DataType.INTEGER, LessThanOrEqualIntsEvaluator.Factory::new),
+        Map.entry(DataType.DOUBLE, LessThanOrEqualDoublesEvaluator.Factory::new),
+        Map.entry(DataType.LONG, LessThanOrEqualLongsEvaluator.Factory::new),
+        Map.entry(DataType.UNSIGNED_LONG, LessThanOrEqualLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATETIME, LessThanOrEqualLongsEvaluator.Factory::new),
+        Map.entry(DataType.KEYWORD, LessThanOrEqualKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.TEXT, LessThanOrEqualKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.VERSION, LessThanOrEqualKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.IP, LessThanOrEqualKeywordsEvaluator.Factory::new)
     );
 
     public LessThanOrEqual(Source source, Expression left, Expression right) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java
index 788c2396fa626..0a60a6da818c1 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 
 import java.time.ZoneId;
@@ -21,20 +20,20 @@
 
 public class NotEquals extends EsqlBinaryComparison implements Negatable<EsqlBinaryComparison> {
     private static final Map<DataType, EsqlArithmeticOperation.BinaryEvaluator> evaluatorMap = Map.ofEntries(
-        Map.entry(DataTypes.BOOLEAN, NotEqualsBoolsEvaluator.Factory::new),
-        Map.entry(DataTypes.INTEGER, NotEqualsIntsEvaluator.Factory::new),
-        Map.entry(DataTypes.DOUBLE, NotEqualsDoublesEvaluator.Factory::new),
-        Map.entry(DataTypes.LONG, NotEqualsLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.UNSIGNED_LONG, NotEqualsLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.DATETIME, NotEqualsLongsEvaluator.Factory::new),
-        Map.entry(DataTypes.GEO_POINT, NotEqualsGeometriesEvaluator.Factory::new),
-        Map.entry(DataTypes.CARTESIAN_POINT, NotEqualsGeometriesEvaluator.Factory::new),
-        Map.entry(DataTypes.GEO_SHAPE, NotEqualsGeometriesEvaluator.Factory::new),
-        Map.entry(DataTypes.CARTESIAN_SHAPE, NotEqualsGeometriesEvaluator.Factory::new),
-        Map.entry(DataTypes.KEYWORD, NotEqualsKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.TEXT, NotEqualsKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.VERSION, NotEqualsKeywordsEvaluator.Factory::new),
-        Map.entry(DataTypes.IP, NotEqualsKeywordsEvaluator.Factory::new)
+        Map.entry(DataType.BOOLEAN, NotEqualsBoolsEvaluator.Factory::new),
+        Map.entry(DataType.INTEGER, NotEqualsIntsEvaluator.Factory::new),
+        Map.entry(DataType.DOUBLE, NotEqualsDoublesEvaluator.Factory::new),
+        Map.entry(DataType.LONG, NotEqualsLongsEvaluator.Factory::new),
+        Map.entry(DataType.UNSIGNED_LONG, NotEqualsLongsEvaluator.Factory::new),
+        Map.entry(DataType.DATETIME, NotEqualsLongsEvaluator.Factory::new),
+        Map.entry(DataType.GEO_POINT, NotEqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataType.CARTESIAN_POINT, NotEqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataType.GEO_SHAPE, NotEqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataType.CARTESIAN_SHAPE, NotEqualsGeometriesEvaluator.Factory::new),
+        Map.entry(DataType.KEYWORD, NotEqualsKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.TEXT, NotEqualsKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.VERSION, NotEqualsKeywordsEvaluator.Factory::new),
+        Map.entry(DataType.IP, NotEqualsKeywordsEvaluator.Factory::new)
     );
 
     public NotEquals(Source source, Expression left, Expression right) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index 6b5e91376a335..bad07bea9926b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -213,8 +213,8 @@
 import java.util.function.Function;
 
 import static java.util.Map.entry;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
 import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.Entry.of;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
index 2bcbbec1a8982..66bd69529f34e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.xpack.esql.core.rule.ParameterizedRuleExecutor;
 import org.elasticsearch.xpack.esql.core.rule.Rule;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
@@ -136,7 +135,7 @@ private LogicalPlan missingToNull(LogicalPlan plan, SearchStats stats) {
             else if (plan instanceof Project project) {
                 var projections = project.projections();
                 List<NamedExpression> newProjections = new ArrayList<>(projections.size());
-                Map<DataType, Alias> nullLiteral = Maps.newLinkedHashMapWithExpectedSize(DataTypes.types().size());
+                Map<DataType, Alias> nullLiteral = Maps.newLinkedHashMapWithExpectedSize(DataType.types().size());
 
                 for (NamedExpression projection : projections) {
                     if (projection instanceof FieldAttribute f && stats.exists(f.qualifiedName()) == false) {
@@ -276,7 +275,7 @@ protected void aggOutput(NamedExpression agg, AggregateFunction aggFunc, BlockFa
             for (Attribute o : output) {
                 DataType dataType = o.dataType();
                 // boolean right now is used for the internal #seen so always return true
-                var value = dataType == DataTypes.BOOLEAN ? true
+                var value = dataType == DataType.BOOLEAN ? true
                     // look for count(literal) with literal != null
                     : aggFunc instanceof Count count && (count.foldable() == false || count.fold() != null) ? 0L
                     // otherwise nullify
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
index d1eb8c32dfcb0..5eb024d410992 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java
@@ -34,7 +34,7 @@
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.rule.ParameterizedRuleExecutor;
 import org.elasticsearch.xpack.esql.core.rule.Rule;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.Queries;
 import org.elasticsearch.xpack.esql.core.util.Queries.Clause;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
@@ -252,7 +252,7 @@ public static boolean canPushToSource(Expression exp, Predicate<FieldAttribute>
             } else if (exp instanceof UnaryScalarFunction usf) {
                 if (usf instanceof RegexMatch<?> || usf instanceof IsNull || usf instanceof IsNotNull) {
                     if (usf instanceof IsNull || usf instanceof IsNotNull) {
-                        if (usf.field() instanceof FieldAttribute fa && fa.dataType().equals(DataTypes.TEXT)) {
+                        if (usf.field() instanceof FieldAttribute fa && fa.dataType().equals(DataType.TEXT)) {
                             return true;
                         }
                     }
@@ -444,7 +444,7 @@ public static boolean hasIdenticalDelegate(FieldAttribute attr, SearchStats stat
 
     public static boolean isPushableFieldAttribute(Expression exp, Predicate<FieldAttribute> hasIdenticalDelegate) {
         if (exp instanceof FieldAttribute fa && fa.getExactInfo().hasExact() && isAggregatable(fa)) {
-            return fa.dataType() != DataTypes.TEXT || hasIdenticalDelegate.test(fa);
+            return fa.dataType() != DataType.TEXT || hasIdenticalDelegate.test(fa);
         }
         return false;
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
index 7beb3aca05e74..62a4a25445ac7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
@@ -47,7 +47,7 @@
 import org.elasticsearch.xpack.esql.core.rule.ParameterizedRuleExecutor;
 import org.elasticsearch.xpack.esql.core.rule.Rule;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
@@ -1286,7 +1286,7 @@ static class AddDefaultTopN extends ParameterizedOptimizerRule<LogicalPlan, Logi
         @Override
         protected LogicalPlan rule(LogicalPlan plan, LogicalOptimizerContext context) {
             if (plan instanceof UnaryPlan unary && unary.child() instanceof OrderBy order && order.child() instanceof EsRelation relation) {
-                var limit = new Literal(plan.source(), context.configuration().resultTruncationMaxSize(), DataTypes.INTEGER);
+                var limit = new Literal(plan.source(), context.configuration().resultTruncationMaxSize(), DataType.INTEGER);
                 return unary.replaceChild(new TopN(plan.source(), relation, order.order(), limit));
             }
             return plan;
@@ -1756,7 +1756,7 @@ private static LogicalPlan normalize(Aggregate aggregate, AttributeMap<Expressio
                             if (fold != null && StringUtils.WILDCARD.equals(fold) == false) {
                                 changed.set(true);
                                 var source = count.source();
-                                af = new Count(source, new Literal(source, StringUtils.WILDCARD, DataTypes.KEYWORD));
+                                af = new Count(source, new Literal(source, StringUtils.WILDCARD, DataType.KEYWORD));
                             }
                         }
                     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
index 6159cb42e29ec..fdddbfc837a51 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.plan.QueryPlan;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
@@ -336,14 +336,14 @@ private static Expression propagate(And and) {
                 } else if (ex instanceof Equals otherEq) {
                     // equals on different values evaluate to FALSE
                     // ignore date/time fields as equality comparison might actually be a range check
-                    if (otherEq.right().foldable() && DataTypes.isDateTime(otherEq.left().dataType()) == false) {
+                    if (otherEq.right().foldable() && DataType.isDateTime(otherEq.left().dataType()) == false) {
                         for (BinaryComparison eq : equals) {
                             if (otherEq.left().semanticEquals(eq.left())) {
                                 Integer comp = BinaryComparison.compare(eq.right().fold(), otherEq.right().fold());
                                 if (comp != null) {
                                     // var cannot be equal to two different values at the same time
                                     if (comp != 0) {
-                                        return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                        return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                                     }
                                 }
                             }
@@ -389,7 +389,7 @@ private static Expression propagate(And and) {
                             compare > 0 ||
                             // eq matches the boundary but should not be included
                                 (compare == 0 && range.includeLower() == false))) {
-                                return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                             }
                         }
                         if (range.upper().foldable()) {
@@ -399,7 +399,7 @@ private static Expression propagate(And and) {
                             compare < 0 ||
                             // eq matches the boundary but should not be included
                                 (compare == 0 && range.includeUpper() == false))) {
-                                return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                             }
                         }
 
@@ -416,7 +416,7 @@ private static Expression propagate(And and) {
                         Integer comp = BinaryComparison.compare(eqValue, neq.right().fold());
                         if (comp != null) {
                             if (comp == 0) { // clashing and conflicting: a = 1 AND a != 1
-                                return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                             } else { // clashing and redundant: a = 1 AND a != 2
                                 iter.remove();
                                 changed = true;
@@ -434,12 +434,12 @@ private static Expression propagate(And and) {
                             if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { // a = 2 AND a </<= ?
                                 if ((compare == 0 && bc instanceof LessThan) || // a = 2 AND a < 2
                                     0 < compare) { // a = 2 AND a </<= 1
-                                    return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                    return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                                 }
                             } else if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) { // a = 2 AND a >/>= ?
                                 if ((compare == 0 && bc instanceof GreaterThan) || // a = 2 AND a > 2
                                     compare < 0) { // a = 2 AND a >/>= 3
-                                    return new Literal(and.source(), Boolean.FALSE, DataTypes.BOOLEAN);
+                                    return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
                                 }
                             }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java
index 9a7ee0a587335..0d3aaaa3a9d47 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SimplifyComparisonsArithmetics.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub;
 
@@ -148,7 +147,7 @@ final boolean isUnsafe(BiFunction<DataType, DataType, Boolean> typesCompatible)
         final Expression apply() {
             // force float point folding for FlP field
             Literal bcl = operation.dataType().isRational()
-                ? new Literal(bcLiteral.source(), ((Number) bcLiteral.value()).doubleValue(), DataTypes.DOUBLE)
+                ? new Literal(bcLiteral.source(), ((Number) bcLiteral.value()).doubleValue(), DataType.DOUBLE)
                 : bcLiteral;
 
             Expression bcRightExpression = ((BinaryComparisonInversible) operation).binaryComparisonInverse()
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
index a94d449b9e401..a32ae9167aeb2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.DateUtils;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.Order;
@@ -78,8 +77,8 @@
 import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.source;
 import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.typedParsing;
 import static org.elasticsearch.xpack.esql.core.parser.ParserUtils.visitList;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD;
@@ -125,7 +124,7 @@ protected List<Expression> expressions(List<? extends ParserRuleContext> context
     @Override
     public Literal visitBooleanValue(EsqlBaseParser.BooleanValueContext ctx) {
         Source source = source(ctx);
-        return new Literal(source, ctx.TRUE() != null, DataTypes.BOOLEAN);
+        return new Literal(source, ctx.TRUE() != null, DataType.BOOLEAN);
     }
 
     @Override
@@ -134,7 +133,7 @@ public Literal visitDecimalValue(EsqlBaseParser.DecimalValueContext ctx) {
         String text = ctx.getText();
 
         try {
-            return new Literal(source, StringUtils.parseDouble(text), DataTypes.DOUBLE);
+            return new Literal(source, StringUtils.parseDouble(text), DataType.DOUBLE);
         } catch (InvalidArgumentException iae) {
             throw new ParsingException(source, iae.getMessage());
         }
@@ -151,7 +150,7 @@ public Literal visitIntegerValue(EsqlBaseParser.IntegerValueContext ctx) {
         } catch (InvalidArgumentException siae) {
             // if it's too large, then quietly try to parse as a float instead
             try {
-                return new Literal(source, EsqlDataTypeConverter.stringToDouble(text), DataTypes.DOUBLE);
+                return new Literal(source, EsqlDataTypeConverter.stringToDouble(text), DataType.DOUBLE);
             } catch (InvalidArgumentException ignored) {}
 
             throw new ParsingException(source, siae.getMessage());
@@ -161,13 +160,13 @@ public Literal visitIntegerValue(EsqlBaseParser.IntegerValueContext ctx) {
         DataType type;
         if (number instanceof BigInteger bi) {
             val = asLongUnsigned(bi);
-            type = DataTypes.UNSIGNED_LONG;
+            type = DataType.UNSIGNED_LONG;
         } else if (number.intValue() == number.longValue()) { // try to downsize to int if possible (since that's the most common type)
             val = number.intValue();
-            type = DataTypes.INTEGER;
+            type = DataType.INTEGER;
         } else {
             val = number.longValue();
-            type = DataTypes.LONG;
+            type = DataType.LONG;
         }
         return new Literal(source, val, type);
     }
@@ -176,25 +175,23 @@ public Literal visitIntegerValue(EsqlBaseParser.IntegerValueContext ctx) {
     public Object visitNumericArrayLiteral(EsqlBaseParser.NumericArrayLiteralContext ctx) {
         Source source = source(ctx);
         List<Literal> numbers = visitList(this, ctx.numericValue(), Literal.class);
-        if (numbers.stream().anyMatch(l -> l.dataType() == DataTypes.DOUBLE)) {
-            return new Literal(source, mapNumbers(numbers, (no, dt) -> no.doubleValue()), DataTypes.DOUBLE);
+        if (numbers.stream().anyMatch(l -> l.dataType() == DataType.DOUBLE)) {
+            return new Literal(source, mapNumbers(numbers, (no, dt) -> no.doubleValue()), DataType.DOUBLE);
         }
-        if (numbers.stream().anyMatch(l -> l.dataType() == DataTypes.UNSIGNED_LONG)) {
+        if (numbers.stream().anyMatch(l -> l.dataType() == DataType.UNSIGNED_LONG)) {
             return new Literal(
                 source,
                 mapNumbers(
                     numbers,
-                    (no, dt) -> dt == DataTypes.UNSIGNED_LONG
-                        ? no.longValue()
-                        : bigIntegerToUnsignedLong(BigInteger.valueOf(no.longValue()))
+                    (no, dt) -> dt == DataType.UNSIGNED_LONG ? no.longValue() : bigIntegerToUnsignedLong(BigInteger.valueOf(no.longValue()))
                 ),
-                DataTypes.UNSIGNED_LONG
+                DataType.UNSIGNED_LONG
             );
         }
-        if (numbers.stream().anyMatch(l -> l.dataType() == DataTypes.LONG)) {
-            return new Literal(source, mapNumbers(numbers, (no, dt) -> no.longValue()), DataTypes.LONG);
+        if (numbers.stream().anyMatch(l -> l.dataType() == DataType.LONG)) {
+            return new Literal(source, mapNumbers(numbers, (no, dt) -> no.longValue()), DataType.LONG);
         }
-        return new Literal(source, mapNumbers(numbers, (no, dt) -> no.intValue()), DataTypes.INTEGER);
+        return new Literal(source, mapNumbers(numbers, (no, dt) -> no.intValue()), DataType.INTEGER);
     }
 
     private List<Object> mapNumbers(List<Literal> numbers, BiFunction<Number, DataType, Object> map) {
@@ -203,12 +200,12 @@ private List<Object> mapNumbers(List<Literal> numbers, BiFunction<Number, DataTy
 
     @Override
     public Object visitBooleanArrayLiteral(EsqlBaseParser.BooleanArrayLiteralContext ctx) {
-        return visitArrayLiteral(ctx, ctx.booleanValue(), DataTypes.BOOLEAN);
+        return visitArrayLiteral(ctx, ctx.booleanValue(), DataType.BOOLEAN);
     }
 
     @Override
     public Object visitStringArrayLiteral(EsqlBaseParser.StringArrayLiteralContext ctx) {
-        return visitArrayLiteral(ctx, ctx.string(), DataTypes.KEYWORD);
+        return visitArrayLiteral(ctx, ctx.string(), DataType.KEYWORD);
     }
 
     private Object visitArrayLiteral(ParserRuleContext ctx, List<? extends ParserRuleContext> contexts, DataType dataType) {
@@ -220,7 +217,7 @@ private Object visitArrayLiteral(ParserRuleContext ctx, List<? extends ParserRul
     @Override
     public Literal visitNullLiteral(EsqlBaseParser.NullLiteralContext ctx) {
         Source source = source(ctx);
-        return new Literal(source, null, DataTypes.NULL);
+        return new Literal(source, null, DataType.NULL);
     }
 
     @Override
@@ -231,7 +228,7 @@ public Literal visitStringLiteral(EsqlBaseParser.StringLiteralContext ctx) {
     @Override
     public Literal visitString(EsqlBaseParser.StringContext ctx) {
         Source source = source(ctx);
-        return new Literal(source, unquoteString(source), DataTypes.KEYWORD);
+        return new Literal(source, unquoteString(source), DataType.KEYWORD);
     }
 
     @Override
@@ -442,7 +439,7 @@ public Object visitQualifiedIntegerLiteral(EsqlBaseParser.QualifiedIntegerLitera
         Source source = source(ctx);
         Literal intLit = typedParsing(this, ctx.integerValue(), Literal.class);
         Number value = (Number) intLit.value();
-        if (intLit.dataType() == DataTypes.UNSIGNED_LONG) {
+        if (intLit.dataType() == DataType.UNSIGNED_LONG) {
             value = unsignedLongAsNumber(value.longValue());
         }
         String qualifier = ctx.UNQUOTED_IDENTIFIER().getText().toLowerCase(Locale.ROOT);
@@ -532,7 +529,7 @@ public Expression visitFunctionExpression(EsqlBaseParser.FunctionExpressionConte
         if ("count".equals(EsqlFunctionRegistry.normalizeName(name))) {
             // to simplify the registration, handle in the parser the special count cases
             if (args.isEmpty() || ctx.ASTERISK() != null) {
-                args = singletonList(new Literal(source(ctx), "*", DataTypes.KEYWORD));
+                args = singletonList(new Literal(source(ctx), "*", DataType.KEYWORD));
             }
         }
         return new UnresolvedFunction(source(ctx), name, FunctionResolutionStrategy.DEFAULT, args);
@@ -554,7 +551,7 @@ public Expression visitInlineCast(EsqlBaseParser.InlineCastContext ctx) {
     public DataType visitToDataType(EsqlBaseParser.ToDataTypeContext ctx) {
         String typeName = visitIdentifier(ctx.identifier());
         DataType dataType = EsqlDataTypes.fromNameOrAlias(typeName);
-        if (dataType == DataTypes.UNSUPPORTED) {
+        if (dataType == DataType.UNSUPPORTED) {
             throw new ParsingException(source(ctx), "Unknown data type named [{}]", typeName);
         }
         return dataType;
@@ -706,7 +703,7 @@ public Object visitInputParam(EsqlBaseParser.InputParamContext ctx) {
         }
         final DataType sourceType;
         try {
-            sourceType = DataTypes.fromJava(param.value);
+            sourceType = DataType.fromJava(param.value);
         } catch (QlIllegalArgumentException ex) {
             throw new ParsingException(
                 ex,
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
index 9d13af4f7eba7..b80edbe895ced 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
@@ -37,7 +37,7 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern;
 import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.MetadataOptionContext;
@@ -168,7 +168,7 @@ public PlanFactory visitDissectCommand(EsqlBaseParser.DissectCommandContext ctx)
                 List<Attribute> keys = new ArrayList<>();
                 for (var x : parser.outputKeys()) {
                     if (x.isEmpty() == false) {
-                        keys.add(new ReferenceAttribute(src, x, DataTypes.KEYWORD));
+                        keys.add(new ReferenceAttribute(src, x, DataType.KEYWORD));
                     }
                 }
                 return new Dissect(src, p, expression(ctx.primaryExpression()), new Dissect.Parser(pattern, appendSeparator, parser), keys);
@@ -300,7 +300,7 @@ public PlanFactory visitWhereCommand(EsqlBaseParser.WhereCommandContext ctx) {
     public PlanFactory visitLimitCommand(EsqlBaseParser.LimitCommandContext ctx) {
         Source source = source(ctx);
         int limit = stringToInt(ctx.INTEGER_LITERAL().getText());
-        return input -> new Limit(source, new Literal(source, limit, DataTypes.INTEGER), input);
+        return input -> new Limit(source, new Literal(source, limit, DataType.INTEGER), input);
     }
 
     @Override
@@ -378,7 +378,7 @@ public PlanFactory visitEnrichCommand(EsqlBaseParser.EnrichCommandContext ctx) {
                 source,
                 p,
                 mode,
-                new Literal(source(ctx.policyName), policyNameString, DataTypes.KEYWORD),
+                new Literal(source(ctx.policyName), policyNameString, DataType.KEYWORD),
                 matchField,
                 null,
                 Map.of(),
@@ -443,7 +443,7 @@ public PlanFactory visitLookupCommand(EsqlBaseParser.LookupCommandContext ctx) {
             }
         });
 
-        Literal tableName = new Literal(source, ctx.tableName.getText(), DataTypes.KEYWORD);
+        Literal tableName = new Literal(source, ctx.tableName.getText(), DataType.KEYWORD);
 
         throw new ParsingException(source, "LOOKUP not yet supported");
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Explain.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Explain.java
index 8500f60bd5d60..86f3e0bdf349a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Explain.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Explain.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 
 import java.util.List;
 import java.util.Objects;
@@ -55,8 +55,8 @@ public Explain(Source source, LogicalPlan query) {
     @Override
     public List<Attribute> output() {
         return List.of(
-            new ReferenceAttribute(Source.EMPTY, "plan", DataTypes.KEYWORD),
-            new ReferenceAttribute(Source.EMPTY, "type", DataTypes.KEYWORD)
+            new ReferenceAttribute(Source.EMPTY, "plan", DataType.KEYWORD),
+            new ReferenceAttribute(Source.EMPTY, "type", DataType.KEYWORD)
         );
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java
index d9f28eff34ad3..5a85e385da8ef 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Grok.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.NamedExpressions;
 import org.elasticsearch.xpack.esql.parser.ParsingException;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -45,12 +44,12 @@ private List<Attribute> extractedFields() {
 
         private static DataType toDataType(GrokCaptureType type) {
             return switch (type) {
-                case STRING -> DataTypes.KEYWORD;
-                case INTEGER -> DataTypes.INTEGER;
-                case LONG -> DataTypes.LONG;
-                case FLOAT -> DataTypes.FLOAT;
-                case DOUBLE -> DataTypes.DOUBLE;
-                case BOOLEAN -> DataTypes.BOOLEAN;
+                case STRING -> DataType.KEYWORD;
+                case INTEGER -> DataType.INTEGER;
+                case LONG -> DataType.LONG;
+                case FLOAT -> DataType.FLOAT;
+                case DOUBLE -> DataType.DOUBLE;
+                case BOOLEAN -> DataType.BOOLEAN;
             };
         }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java
index 32159ac3caea8..6356b2644e67a 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java
@@ -25,8 +25,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 
 public class MetaFunctions extends LeafPlan {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/show/ShowInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/show/ShowInfo.java
index 9ff97fe47d2a9..4867d8ca77a39 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/show/ShowInfo.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/show/ShowInfo.java
@@ -19,7 +19,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 
 public class ShowInfo extends LeafPlan {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
index 30986f9c626da..fc43f1002d112 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.NodeUtils;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 
 import java.util.List;
@@ -28,10 +28,10 @@
 import java.util.Objects;
 
 public class EsQueryExec extends LeafExec implements EstimatesRowSize {
-    static final EsField DOC_ID_FIELD = new EsField("_doc", DataTypes.DOC_DATA_TYPE, Map.of(), false);
-    static final EsField TSID_FIELD = new EsField("_tsid", DataTypes.TSID_DATA_TYPE, Map.of(), true);
-    static final EsField TIMESTAMP_FIELD = new EsField("@timestamp", DataTypes.DATETIME, Map.of(), true);
-    static final EsField INTERVAL_FIELD = new EsField("@timestamp_interval", DataTypes.DATETIME, Map.of(), true);
+    static final EsField DOC_ID_FIELD = new EsField("_doc", DataType.DOC_DATA_TYPE, Map.of(), false);
+    static final EsField TSID_FIELD = new EsField("_tsid", DataType.TSID_DATA_TYPE, Map.of(), true);
+    static final EsField TIMESTAMP_FIELD = new EsField("@timestamp", DataType.DATETIME, Map.of(), true);
+    static final EsField INTERVAL_FIELD = new EsField("@timestamp_interval", DataType.DATETIME, Map.of(), true);
 
     private final EsIndex index;
     private final IndexMode indexMode;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
index 05bc7d6b1bb8a..68e6ea4d6cadb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.xpack.esql.core.expression.function.Function;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
 import org.elasticsearch.xpack.esql.expression.function.aggregate.CountDistinct;
@@ -44,8 +43,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
 
 final class AggregateMapper {
 
@@ -228,11 +227,11 @@ private static Stream<NamedExpression> isToNE(List<IntermediateStateDesc> interm
     // defaults to aggstate, but we'll eventually be able to remove this
     private static DataType toDataType(ElementType elementType) {
         return switch (elementType) {
-            case BOOLEAN -> DataTypes.BOOLEAN;
-            case BYTES_REF -> DataTypes.KEYWORD;
-            case INT -> DataTypes.INTEGER;
-            case LONG -> DataTypes.LONG;
-            case DOUBLE -> DataTypes.DOUBLE;
+            case BOOLEAN -> DataType.BOOLEAN;
+            case BYTES_REF -> DataType.KEYWORD;
+            case INT -> DataType.INTEGER;
+            case LONG -> DataType.LONG;
+            case DOUBLE -> DataType.DOUBLE;
             default -> throw new EsqlIllegalArgumentException("unsupported agg type: " + elementType);
         };
     }
@@ -242,18 +241,18 @@ private static String dataTypeToString(DataType type, Class<?> aggClass) {
         if (aggClass == Count.class) {
             return "";  // no type distinction
         }
-        if (type.equals(DataTypes.BOOLEAN)) {
+        if (type.equals(DataType.BOOLEAN)) {
             return "Boolean";
-        } else if (type.equals(DataTypes.INTEGER)) {
+        } else if (type.equals(DataType.INTEGER)) {
             return "Int";
-        } else if (type.equals(DataTypes.LONG) || type.equals(DataTypes.DATETIME)) {
+        } else if (type.equals(DataType.LONG) || type.equals(DataType.DATETIME)) {
             return "Long";
-        } else if (type.equals(DataTypes.DOUBLE)) {
+        } else if (type.equals(DataType.DOUBLE)) {
             return "Double";
-        } else if (type.equals(DataTypes.KEYWORD)
-            || type.equals(DataTypes.IP)
-            || type.equals(DataTypes.VERSION)
-            || type.equals(DataTypes.TEXT)) {
+        } else if (type.equals(DataType.KEYWORD)
+            || type.equals(DataType.IP)
+            || type.equals(DataType.VERSION)
+            || type.equals(DataType.TEXT)) {
                 return "BytesRef";
             } else if (type.equals(GEO_POINT)) {
                 return "GeoPoint";
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
index 8873bd770a84e..349483116a0a8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.xpack.esql.core.querydsl.query.TermsQuery;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.Check;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction;
@@ -54,9 +53,9 @@
 import java.util.List;
 import java.util.Set;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.HOUR_MINUTE_SECOND;
@@ -203,7 +202,7 @@ static Query translate(BinaryComparison bc, TranslatorHandler handler) {
             }
 
             ZoneId zoneId = null;
-            if (DataTypes.isDateTime(attribute.dataType())) {
+            if (DataType.isDateTime(attribute.dataType())) {
                 zoneId = bc.zoneId();
             }
             if (bc instanceof GreaterThan) {
@@ -301,28 +300,28 @@ private static boolean isInRange(DataType numericFieldDataType, DataType valueDa
             // Determine min/max for dataType. Use BigDecimals as doubles will have rounding errors for long/ulong.
             BigDecimal minValue;
             BigDecimal maxValue;
-            if (numericFieldDataType == DataTypes.BYTE) {
+            if (numericFieldDataType == DataType.BYTE) {
                 minValue = BigDecimal.valueOf(Byte.MIN_VALUE);
                 maxValue = BigDecimal.valueOf(Byte.MAX_VALUE);
-            } else if (numericFieldDataType == DataTypes.SHORT) {
+            } else if (numericFieldDataType == DataType.SHORT) {
                 minValue = BigDecimal.valueOf(Short.MIN_VALUE);
                 maxValue = BigDecimal.valueOf(Short.MAX_VALUE);
-            } else if (numericFieldDataType == DataTypes.INTEGER) {
+            } else if (numericFieldDataType == DataType.INTEGER) {
                 minValue = BigDecimal.valueOf(Integer.MIN_VALUE);
                 maxValue = BigDecimal.valueOf(Integer.MAX_VALUE);
-            } else if (numericFieldDataType == DataTypes.LONG) {
+            } else if (numericFieldDataType == DataType.LONG) {
                 minValue = BigDecimal.valueOf(Long.MIN_VALUE);
                 maxValue = BigDecimal.valueOf(Long.MAX_VALUE);
-            } else if (numericFieldDataType == DataTypes.UNSIGNED_LONG) {
+            } else if (numericFieldDataType == DataType.UNSIGNED_LONG) {
                 minValue = BigDecimal.ZERO;
                 maxValue = UNSIGNED_LONG_MAX;
-            } else if (numericFieldDataType == DataTypes.HALF_FLOAT) {
+            } else if (numericFieldDataType == DataType.HALF_FLOAT) {
                 minValue = HALF_FLOAT_MAX.negate();
                 maxValue = HALF_FLOAT_MAX;
-            } else if (numericFieldDataType == DataTypes.FLOAT) {
+            } else if (numericFieldDataType == DataType.FLOAT) {
                 minValue = BigDecimal.valueOf(-Float.MAX_VALUE);
                 maxValue = BigDecimal.valueOf(Float.MAX_VALUE);
-            } else if (numericFieldDataType == DataTypes.DOUBLE || numericFieldDataType == DataTypes.SCALED_FLOAT) {
+            } else if (numericFieldDataType == DataType.DOUBLE || numericFieldDataType == DataType.SCALED_FLOAT) {
                 // Scaled floats are represented as doubles in ESQL.
                 minValue = BigDecimal.valueOf(-Double.MAX_VALUE);
                 maxValue = BigDecimal.valueOf(Double.MAX_VALUE);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
index 39b641a872c58..cc28839fd6575 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.core.util.Queries;
 import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext;
@@ -240,37 +239,37 @@ public static ElementType toElementType(DataType dataType) {
      * For example, spatial types can be extracted into doc-values under specific conditions, otherwise they extract as BytesRef.
      */
     public static ElementType toElementType(DataType dataType, MappedFieldType.FieldExtractPreference fieldExtractPreference) {
-        if (dataType == DataTypes.LONG
-            || dataType == DataTypes.DATETIME
-            || dataType == DataTypes.UNSIGNED_LONG
-            || dataType == DataTypes.COUNTER_LONG) {
+        if (dataType == DataType.LONG
+            || dataType == DataType.DATETIME
+            || dataType == DataType.UNSIGNED_LONG
+            || dataType == DataType.COUNTER_LONG) {
             return ElementType.LONG;
         }
-        if (dataType == DataTypes.INTEGER || dataType == DataTypes.COUNTER_INTEGER) {
+        if (dataType == DataType.INTEGER || dataType == DataType.COUNTER_INTEGER) {
             return ElementType.INT;
         }
-        if (dataType == DataTypes.DOUBLE || dataType == DataTypes.COUNTER_DOUBLE) {
+        if (dataType == DataType.DOUBLE || dataType == DataType.COUNTER_DOUBLE) {
             return ElementType.DOUBLE;
         }
         // unsupported fields are passed through as a BytesRef
-        if (dataType == DataTypes.KEYWORD
-            || dataType == DataTypes.TEXT
-            || dataType == DataTypes.IP
-            || dataType == DataTypes.SOURCE
-            || dataType == DataTypes.VERSION
-            || dataType == DataTypes.UNSUPPORTED) {
+        if (dataType == DataType.KEYWORD
+            || dataType == DataType.TEXT
+            || dataType == DataType.IP
+            || dataType == DataType.SOURCE
+            || dataType == DataType.VERSION
+            || dataType == DataType.UNSUPPORTED) {
             return ElementType.BYTES_REF;
         }
-        if (dataType == DataTypes.NULL) {
+        if (dataType == DataType.NULL) {
             return ElementType.NULL;
         }
-        if (dataType == DataTypes.BOOLEAN) {
+        if (dataType == DataType.BOOLEAN) {
             return ElementType.BOOLEAN;
         }
-        if (dataType == DataTypes.DOC_DATA_TYPE) {
+        if (dataType == DataType.DOC_DATA_TYPE) {
             return ElementType.DOC;
         }
-        if (dataType == DataTypes.TSID_DATA_TYPE) {
+        if (dataType == DataType.TSID_DATA_TYPE) {
             return ElementType.BYTES_REF;
         }
         if (EsqlDataTypes.isSpatialPoint(dataType)) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
index 9b6d392b5c899..23d2126e39706 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java
@@ -40,7 +40,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
 
 public class SpatialRelatesQuery extends Query {
     private final String field;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlIndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlIndexResolver.java
index 6108fbe7d0cae..f973983e47f39 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlIndexResolver.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlIndexResolver.java
@@ -37,11 +37,11 @@
 import java.util.TreeMap;
 import java.util.TreeSet;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.OBJECT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSUPPORTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
 
 public class EsqlIndexResolver {
     private final Client client;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
index ce4bf5d6143a3..10116bb24e6b9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
@@ -35,7 +35,7 @@
 import org.elasticsearch.xpack.esql.core.index.MappingException;
 import org.elasticsearch.xpack.esql.core.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
 import org.elasticsearch.xpack.esql.core.util.Holder;
@@ -291,7 +291,7 @@ private void assertSameMappings(String prefix, Map<String, EsField> fromQl, Map<
                              * we don't actually use it in ESQL and the EsqlIndexResolver doesn't
                              * produce exactly the same result.
                              */
-                            if (qlField.getDataType().equals(DataTypes.UNSUPPORTED) == false
+                            if (qlField.getDataType().equals(DataType.UNSUPPORTED) == false
                                 && qlField.getName().equals(esqlField.getName()) == false
                             // QL uses full paths for unsupported fields. ESQL does not. This particular difference is fine.
                             ) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
index 7ce2d012bf76b..cc2525799224b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.core.type.Converter;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypeConverter;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
@@ -54,27 +53,27 @@
 import java.util.function.Function;
 
 import static java.util.Map.entry;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isPrimitive;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isString;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeDoubleToLong;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToInt;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToLong;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToUnsignedLong;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NULL;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isPrimitive;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isString;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.ONE_AS_UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.ZERO_AS_UNSIGNED_LONG;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
@@ -122,34 +121,34 @@ public static boolean canConvert(DataType from, DataType to) {
     public static Converter converterFor(DataType from, DataType to) {
         // TODO move EXPRESSION_TO_LONG here if there is no regression
         if (isString(from)) {
-            if (to == DataTypes.DATETIME) {
+            if (to == DataType.DATETIME) {
                 return EsqlConverter.STRING_TO_DATETIME;
             }
-            if (to == DataTypes.IP) {
+            if (to == DataType.IP) {
                 return EsqlConverter.STRING_TO_IP;
             }
-            if (to == DataTypes.VERSION) {
+            if (to == DataType.VERSION) {
                 return EsqlConverter.STRING_TO_VERSION;
             }
-            if (to == DataTypes.DOUBLE) {
+            if (to == DataType.DOUBLE) {
                 return EsqlConverter.STRING_TO_DOUBLE;
             }
-            if (to == DataTypes.LONG) {
+            if (to == DataType.LONG) {
                 return EsqlConverter.STRING_TO_LONG;
             }
-            if (to == DataTypes.INTEGER) {
+            if (to == DataType.INTEGER) {
                 return EsqlConverter.STRING_TO_INT;
             }
-            if (to == DataTypes.BOOLEAN) {
+            if (to == DataType.BOOLEAN) {
                 return EsqlConverter.STRING_TO_BOOLEAN;
             }
             if (EsqlDataTypes.isSpatial(to)) {
                 return EsqlConverter.STRING_TO_SPATIAL;
             }
-            if (to == DataTypes.TIME_DURATION) {
+            if (to == DataType.TIME_DURATION) {
                 return EsqlConverter.STRING_TO_TIME_DURATION;
             }
-            if (to == DataTypes.DATE_PERIOD) {
+            if (to == DataType.DATE_PERIOD) {
                 return EsqlConverter.STRING_TO_DATE_PERIOD;
             }
         }
@@ -188,15 +187,15 @@ public static TemporalAmount parseTemporalAmount(Object val, DataType expectedTy
         if ((value.isEmpty() || qualifier.isEmpty()) == false) {
             try {
                 TemporalAmount result = parseTemporalAmout(Integer.parseInt(value.toString()), qualifier.toString(), Source.EMPTY);
-                if (DataTypes.DATE_PERIOD == expectedType && result instanceof Period
-                    || DataTypes.TIME_DURATION == expectedType && result instanceof Duration) {
+                if (DataType.DATE_PERIOD == expectedType && result instanceof Period
+                    || DataType.TIME_DURATION == expectedType && result instanceof Duration) {
                     return result;
                 }
-                if (result instanceof Period && expectedType == DataTypes.TIME_DURATION) {
-                    errorMessage += ", did you mean " + DataTypes.DATE_PERIOD + "?";
+                if (result instanceof Period && expectedType == DataType.TIME_DURATION) {
+                    errorMessage += ", did you mean " + DataType.DATE_PERIOD + "?";
                 }
-                if (result instanceof Duration && expectedType == DataTypes.DATE_PERIOD) {
-                    errorMessage += ", did you mean " + DataTypes.TIME_DURATION + "?";
+                if (result instanceof Duration && expectedType == DataType.DATE_PERIOD) {
+                    errorMessage += ", did you mean " + DataType.TIME_DURATION + "?";
                 }
             } catch (NumberFormatException ex) {
                 // wrong pattern
@@ -431,8 +430,8 @@ public static long booleanToUnsignedLong(boolean number) {
 
     public enum EsqlConverter implements Converter {
 
-        STRING_TO_DATE_PERIOD(x -> EsqlDataTypeConverter.parseTemporalAmount(x, DataTypes.DATE_PERIOD)),
-        STRING_TO_TIME_DURATION(x -> EsqlDataTypeConverter.parseTemporalAmount(x, DataTypes.TIME_DURATION)),
+        STRING_TO_DATE_PERIOD(x -> EsqlDataTypeConverter.parseTemporalAmount(x, DataType.DATE_PERIOD)),
+        STRING_TO_TIME_DURATION(x -> EsqlDataTypeConverter.parseTemporalAmount(x, DataType.TIME_DURATION)),
         STRING_TO_CHRONO_FIELD(EsqlDataTypeConverter::stringToChrono),
         STRING_TO_DATETIME(x -> EsqlDataTypeConverter.dateTimeToLong((String) x)),
         STRING_TO_IP(x -> EsqlDataTypeConverter.stringToIP((String) x)),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java
index 936aeab569e5f..dc680e5305842 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java
@@ -10,14 +10,13 @@
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.DataTypeRegistry;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.util.Collection;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isDateTime;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isDateTimeOrTemporal;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrDatePeriod;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrTemporalAmount;
@@ -31,7 +30,7 @@ private EsqlDataTypeRegistry() {}
 
     @Override
     public Collection<DataType> dataTypes() {
-        return DataTypes.types();
+        return DataType.types();
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java
index 893dcfe9103ac..e48b46758f36c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java
@@ -8,7 +8,6 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 import java.util.Collections;
 import java.util.Locale;
@@ -17,45 +16,45 @@
 
 import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toUnmodifiableMap;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BYTE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.HALF_FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NESTED;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.NULL;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.OBJECT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.SCALED_FLOAT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.SHORT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.SOURCE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSUPPORTED;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isNull;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.HALF_FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
+import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SCALED_FLOAT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.SOURCE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isNull;
 
 public final class EsqlDataTypes {
 
-    private static final Map<String, DataType> NAME_TO_TYPE = DataTypes.types()
+    private static final Map<String, DataType> NAME_TO_TYPE = DataType.types()
         .stream()
         .collect(toUnmodifiableMap(DataType::typeName, t -> t));
 
     private static final Map<String, DataType> ES_TO_TYPE;
 
     static {
-        Map<String, DataType> map = DataTypes.types().stream().filter(e -> e.esType() != null).collect(toMap(DataType::esType, t -> t));
+        Map<String, DataType> map = DataType.types().stream().filter(e -> e.esType() != null).collect(toMap(DataType::esType, t -> t));
         // ES calls this 'point', but ESQL calls it 'cartesian_point'
-        map.put("point", DataTypes.CARTESIAN_POINT);
-        map.put("shape", DataTypes.CARTESIAN_SHAPE);
+        map.put("point", DataType.CARTESIAN_POINT);
+        map.put("shape", DataType.CARTESIAN_SHAPE);
         ES_TO_TYPE = Collections.unmodifiableMap(map);
     }
 
     private static final Map<String, DataType> NAME_OR_ALIAS_TO_TYPE;
     static {
-        Map<String, DataType> map = DataTypes.types().stream().collect(toMap(DataType::typeName, Function.identity()));
+        Map<String, DataType> map = DataType.types().stream().collect(toMap(DataType::typeName, Function.identity()));
         map.put("bool", BOOLEAN);
         map.put("int", INTEGER);
         map.put("string", KEYWORD);
@@ -105,7 +104,7 @@ public static DataType fromJava(Object value) {
     }
 
     public static boolean isUnsupported(DataType type) {
-        return DataTypes.isUnsupported(type);
+        return DataType.isUnsupported(type);
     }
 
     public static String outputType(DataType type) {
@@ -124,11 +123,11 @@ public static boolean isPrimitive(DataType t) {
     }
 
     public static boolean isDateTimeOrTemporal(DataType t) {
-        return DataTypes.isDateTime(t) || isTemporalAmount(t);
+        return DataType.isDateTime(t) || isTemporalAmount(t);
     }
 
     public static boolean isTemporalAmount(DataType t) {
-        return t == DataTypes.DATE_PERIOD || t == DataTypes.TIME_DURATION;
+        return t == DataType.DATE_PERIOD || t == DataType.TIME_DURATION;
     }
 
     public static boolean isNullOrTemporalAmount(DataType t) {
@@ -136,23 +135,23 @@ public static boolean isNullOrTemporalAmount(DataType t) {
     }
 
     public static boolean isNullOrDatePeriod(DataType t) {
-        return t == DataTypes.DATE_PERIOD || isNull(t);
+        return t == DataType.DATE_PERIOD || isNull(t);
     }
 
     public static boolean isNullOrTimeDuration(DataType t) {
-        return t == DataTypes.TIME_DURATION || isNull(t);
+        return t == DataType.TIME_DURATION || isNull(t);
     }
 
     public static boolean isSpatial(DataType t) {
-        return t == DataTypes.GEO_POINT || t == DataTypes.CARTESIAN_POINT || t == DataTypes.GEO_SHAPE || t == DataTypes.CARTESIAN_SHAPE;
+        return t == DataType.GEO_POINT || t == DataType.CARTESIAN_POINT || t == DataType.GEO_SHAPE || t == DataType.CARTESIAN_SHAPE;
     }
 
     public static boolean isSpatialGeo(DataType t) {
-        return t == DataTypes.GEO_POINT || t == DataTypes.GEO_SHAPE;
+        return t == DataType.GEO_POINT || t == DataType.GEO_SHAPE;
     }
 
     public static boolean isSpatialPoint(DataType t) {
-        return t == DataTypes.GEO_POINT || t == DataTypes.CARTESIAN_POINT;
+        return t == DataType.GEO_POINT || t == DataType.CARTESIAN_POINT;
     }
 
     /**
@@ -201,6 +200,6 @@ public static DataType getCounterType(String typeName) {
     }
 
     public static boolean isCounterType(DataType dt) {
-        return dt == DataTypes.COUNTER_LONG || dt == DataTypes.COUNTER_INTEGER || dt == DataTypes.COUNTER_DOUBLE;
+        return dt == DataType.COUNTER_LONG || dt == DataType.COUNTER_INTEGER || dt == DataType.COUNTER_DOUBLE;
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
index 0ff0b2c9bd9b9..59cabcd459fba 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java
@@ -403,7 +403,7 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception {
         List<Type> columnTypes = coordinatorPlan.output()
             .stream()
             .peek(o -> dataTypes.add(EsqlDataTypes.outputType(o.dataType())))
-            .map(o -> Type.asType(o.dataType().name()))
+            .map(o -> Type.asType(o.dataType().nameUpper()))
             .toList();
 
         List<Driver> drivers = new ArrayList<>();
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
index 317fabf0ac434..5fafafe34bd23 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.Column;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.parser.TypedParamValue;
 
 import java.io.IOException;
@@ -227,7 +227,7 @@ public void testTablesKeyword() throws IOException {
             """;
         EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
         Column c = request.tables().get("a").get("c");
-        assertThat(c.type(), equalTo(DataTypes.KEYWORD));
+        assertThat(c.type(), equalTo(DataType.KEYWORD));
         try (
             BytesRefBlock.Builder builder = new BlockFactory(
                 new NoopCircuitBreaker(CircuitBreaker.REQUEST),
@@ -259,7 +259,7 @@ public void testTablesInteger() throws IOException {
 
         EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
         Column c = request.tables().get("a").get("c");
-        assertThat(c.type(), equalTo(DataTypes.INTEGER));
+        assertThat(c.type(), equalTo(DataType.INTEGER));
         try (
             IntBlock.Builder builder = new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE)
                 .newIntBlockBuilder(10)
@@ -287,7 +287,7 @@ public void testTablesLong() throws IOException {
 
         EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
         Column c = request.tables().get("a").get("c");
-        assertThat(c.type(), equalTo(DataTypes.LONG));
+        assertThat(c.type(), equalTo(DataType.LONG));
         try (
             LongBlock.Builder builder = new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE)
                 .newLongBlockBuilder(10)
@@ -329,15 +329,15 @@ public void testManyTables() throws IOException {
         EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
         assertThat(request.tables().keySet(), hasSize(2));
         Map<String, Column> t1 = request.tables().get("t1");
-        assertThat(t1.get("a").type(), equalTo(DataTypes.LONG));
-        assertThat(t1.get("b").type(), equalTo(DataTypes.LONG));
-        assertThat(t1.get("c").type(), equalTo(DataTypes.KEYWORD));
-        assertThat(t1.get("d").type(), equalTo(DataTypes.LONG));
+        assertThat(t1.get("a").type(), equalTo(DataType.LONG));
+        assertThat(t1.get("b").type(), equalTo(DataType.LONG));
+        assertThat(t1.get("c").type(), equalTo(DataType.KEYWORD));
+        assertThat(t1.get("d").type(), equalTo(DataType.LONG));
         Map<String, Column> t2 = request.tables().get("t2");
-        assertThat(t2.get("a").type(), equalTo(DataTypes.LONG));
-        assertThat(t2.get("b").type(), equalTo(DataTypes.INTEGER));
-        assertThat(t2.get("c").type(), equalTo(DataTypes.LONG));
-        assertThat(t2.get("d").type(), equalTo(DataTypes.LONG));
+        assertThat(t2.get("a").type(), equalTo(DataType.LONG));
+        assertThat(t2.get("b").type(), equalTo(DataType.INTEGER));
+        assertThat(t2.get("c").type(), equalTo(DataType.LONG));
+        assertThat(t2.get("d").type(), equalTo(DataType.LONG));
         assertTablesOnlyValidOnSnapshot(request);
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
index c1fec5d48d23d..9bc792ffe04aa 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java
@@ -47,7 +47,6 @@
 import org.elasticsearch.xpack.core.esql.action.ColumnInfo;
 import org.elasticsearch.xpack.esql.TestBlockFactory;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.elasticsearch.xpack.versionfield.Version;
@@ -125,8 +124,8 @@ EsqlQueryResponse randomResponseAsync(boolean columnar, EsqlQueryResponse.Profil
 
     private ColumnInfo randomColumnInfo() {
         DataType type = randomValueOtherThanMany(
-            t -> false == DataTypes.isPrimitive(t) || t == DataTypes.DATE_PERIOD || t == DataTypes.TIME_DURATION,
-            () -> randomFrom(DataTypes.types())
+            t -> false == DataType.isPrimitive(t) || t == DataType.DATE_PERIOD || t == DataType.TIME_DURATION,
+            () -> randomFrom(DataType.types())
         );
         type = EsqlDataTypes.widenSmallNumericTypes(type);
         return new ColumnInfo(randomAlphaOfLength(10), type.esType());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index 2d9b52f1b433f..fb3c5d737944e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.TypesTests;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
@@ -182,7 +181,7 @@ public void testRowAttributeResolution() {
         var plan = analyzer.analyze(
             new Eval(
                 EMPTY,
-                new Row(EMPTY, List.of(new Alias(EMPTY, "emp_no", new Literal(EMPTY, 1, DataTypes.INTEGER)))),
+                new Row(EMPTY, List.of(new Alias(EMPTY, "emp_no", new Literal(EMPTY, 1, DataType.INTEGER)))),
                 List.of(new Alias(EMPTY, "e", new UnresolvedAttribute(EMPTY, "emp_no")))
             )
         );
@@ -190,7 +189,7 @@ public void testRowAttributeResolution() {
         var limit = as(plan, Limit.class);
         var eval = as(limit.child(), Eval.class);
         assertEquals(1, eval.fields().size());
-        assertEquals(new Alias(EMPTY, "e", new ReferenceAttribute(EMPTY, "emp_no", DataTypes.INTEGER)), eval.fields().get(0));
+        assertEquals(new Alias(EMPTY, "e", new ReferenceAttribute(EMPTY, "emp_no", DataType.INTEGER)), eval.fields().get(0));
 
         assertEquals(2, eval.output().size());
         Attribute empNo = eval.output().get(0);
@@ -233,7 +232,7 @@ public void testProjectBasicPattern() {
         assertProjectionTypes("""
             from test
             | keep first*name
-            """, DataTypes.KEYWORD);
+            """, DataType.KEYWORD);
     }
 
     public void testProjectIncludePattern() {
@@ -294,16 +293,16 @@ public void testNoProjection() {
             """
                 from test
                 """,
-            DataTypes.KEYWORD,
-            DataTypes.INTEGER,
-            DataTypes.KEYWORD,
-            DataTypes.TEXT,
-            DataTypes.TEXT,
-            DataTypes.KEYWORD,
-            DataTypes.INTEGER,
-            DataTypes.KEYWORD,
-            DataTypes.LONG,
-            DataTypes.INTEGER
+            DataType.KEYWORD,
+            DataType.INTEGER,
+            DataType.KEYWORD,
+            DataType.TEXT,
+            DataType.TEXT,
+            DataType.KEYWORD,
+            DataType.INTEGER,
+            DataType.KEYWORD,
+            DataType.LONG,
+            DataType.INTEGER
         );
     }
 
@@ -1661,10 +1660,10 @@ public void testCounterTypes() {
             attributes.keySet(),
             equalTo(Set.of("network.connections", "network.bytes_in", "network.bytes_out", "network.message_in"))
         );
-        assertThat(attributes.get("network.connections").dataType(), equalTo(DataTypes.LONG));
-        assertThat(attributes.get("network.bytes_in").dataType(), equalTo(DataTypes.COUNTER_LONG));
-        assertThat(attributes.get("network.bytes_out").dataType(), equalTo(DataTypes.COUNTER_LONG));
-        assertThat(attributes.get("network.message_in").dataType(), equalTo(DataTypes.COUNTER_DOUBLE));
+        assertThat(attributes.get("network.connections").dataType(), equalTo(DataType.LONG));
+        assertThat(attributes.get("network.bytes_in").dataType(), equalTo(DataType.COUNTER_LONG));
+        assertThat(attributes.get("network.bytes_out").dataType(), equalTo(DataType.COUNTER_LONG));
+        assertThat(attributes.get("network.message_in").dataType(), equalTo(DataType.COUNTER_DOUBLE));
     }
 
     public void testMissingAttributeException_InChainedEval() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
index 4c3a415c5f706..07a364bf9196e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.parser.TypedParamValue;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -20,7 +19,7 @@
 
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.matchesRegex;
@@ -308,7 +307,7 @@ public void testMixedNumericalNonConvertibleTypesInIn() {
     }
 
     public void testUnsignedLongTypeMixInComparisons() {
-        List<String> types = DataTypes.types()
+        List<String> types = DataType.types()
             .stream()
             .filter(dt -> dt.isNumeric() && EsqlDataTypes.isRepresentable(dt) && dt != UNSIGNED_LONG)
             .map(DataType::typeName)
@@ -346,7 +345,7 @@ public void testUnsignedLongTypeMixInComparisons() {
     }
 
     public void testUnsignedLongTypeMixInArithmetics() {
-        List<String> types = DataTypes.types()
+        List<String> types = DataType.types()
             .stream()
             .filter(dt -> dt.isNumeric() && EsqlDataTypes.isRepresentable(dt) && dt != UNSIGNED_LONG)
             .map(DataType::typeName)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperatorTests.java
index dead11069d2e7..107c2af11c4f1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperatorTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichQuerySourceOperatorTests.java
@@ -46,7 +46,7 @@
 import java.util.Map;
 import java.util.Set;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.mockito.Mockito.mock;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/CanonicalTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/CanonicalTests.java
index cbac81700a81a..692ad6f3dcb27 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/CanonicalTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/CanonicalTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod;
@@ -167,7 +166,7 @@ public void testNot() throws Exception {
     }
 
     public void testLiteralHashSorting() throws Exception {
-        DataType type = randomFrom(DataTypes.types());
+        DataType type = randomFrom(DataType.types());
         List<Expression> list = randomList(10, 1024, () -> new Literal(EMPTY, randomInt(), type));
         List<Expression> shuffle = new ArrayList<>(list);
         Collections.shuffle(shuffle, random());
@@ -183,9 +182,9 @@ public void testLiteralHashSorting() throws Exception {
     public void testInManual() throws Exception {
         FieldAttribute value = fieldAttribute();
 
-        Literal a = new Literal(EMPTY, 1, DataTypes.INTEGER);
-        Literal b = new Literal(EMPTY, 2, DataTypes.INTEGER);
-        Literal c = new Literal(EMPTY, 3, DataTypes.INTEGER);
+        Literal a = new Literal(EMPTY, 1, DataType.INTEGER);
+        Literal b = new Literal(EMPTY, 2, DataType.INTEGER);
+        Literal c = new Literal(EMPTY, 3, DataType.INTEGER);
 
         In in = new In(EMPTY, value, asList(a, b, c));
         In anotherIn = new In(EMPTY, value, asList(b, a, c));
@@ -196,7 +195,7 @@ public void testInManual() throws Exception {
 
     public void testIn() throws Exception {
         FieldAttribute value = fieldAttribute();
-        List<Expression> list = randomList(randomInt(1024), () -> new Literal(EMPTY, randomInt(), DataTypes.INTEGER));
+        List<Expression> list = randomList(randomInt(1024), () -> new Literal(EMPTY, randomInt(), DataType.INTEGER));
         In in = new In(EMPTY, value, list);
         List<Expression> shuffledList = new ArrayList<>(list);
         Collections.shuffle(shuffledList, random());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
index 3b7d36015dcbc..20c2b6df9710a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
@@ -49,7 +49,6 @@
 import org.elasticsearch.xpack.esql.core.expression.function.FunctionDefinition;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
@@ -291,7 +290,7 @@ public final void testEvaluate() {
     private Object toJavaObjectUnsignedLongAware(Block block, int position) {
         Object result;
         result = toJavaObject(block, position);
-        if (result != null && testCase.expectedType() == DataTypes.UNSIGNED_LONG) {
+        if (result != null && testCase.expectedType() == DataType.UNSIGNED_LONG) {
             assertThat(result, instanceOf(Long.class));
             result = NumericUtils.unsignedLongAsBigInteger((Long) result);
         }
@@ -547,7 +546,7 @@ public final void testFold() {
         if (testCase.foldingExceptionClass() == null) {
             Object result = nullOptimized.fold();
             // Decode unsigned longs into BigIntegers
-            if (testCase.expectedType() == DataTypes.UNSIGNED_LONG && result != null) {
+            if (testCase.expectedType() == DataType.UNSIGNED_LONG && result != null) {
                 result = NumericUtils.unsignedLongAsBigInteger((Long) result);
             }
             assertThat(result, testCase.getMatcher());
@@ -628,8 +627,8 @@ protected static List<TestCaseSupplier> anyNullIsNull(boolean entirelyNullPreser
         return anyNullIsNull(
             testCaseSuppliers,
             (nullPosition, nullValueDataType, original) -> entirelyNullPreservesType == false
-                && nullValueDataType == DataTypes.NULL
-                && original.getData().size() == 1 ? DataTypes.NULL : original.expectedType(),
+                && nullValueDataType == DataType.NULL
+                && original.getData().size() == 1 ? DataType.NULL : original.expectedType(),
             (nullPosition, nullData, original) -> original
         );
     }
@@ -686,7 +685,7 @@ protected static List<TestCaseSupplier> anyNullIsNull(
 
                 if (firstTimeSeenSignature) {
                     List<DataType> typesWithNull = IntStream.range(0, original.types().size())
-                        .mapToObj(i -> i == finalNullPosition ? DataTypes.NULL : original.types().get(i))
+                        .mapToObj(i -> i == finalNullPosition ? DataType.NULL : original.types().get(i))
                         .toList();
                     boolean newSignature = uniqueSignatures.add(typesWithNull);
                     if (newSignature) {
@@ -698,7 +697,7 @@ protected static List<TestCaseSupplier> anyNullIsNull(
                             return new TestCaseSupplier.TestCase(
                                 data,
                                 equalTo("LiteralsEvaluator[lit=null]"),
-                                expectedType.expectedType(finalNullPosition, DataTypes.NULL, oc),
+                                expectedType.expectedType(finalNullPosition, DataType.NULL, oc),
                                 nullValue(),
                                 null,
                                 oc.getExpectedTypeError(),
@@ -745,7 +744,7 @@ protected static List<TestCaseSupplier> errorsForCasesWithoutExamples(
              * the full combinatorial explosions of all nulls - just a single null.
              * Hopefully <null>, <null> cases will function the same as <null>, <valid>
              * cases.
-             */.filter(types -> types.stream().filter(t -> t == DataTypes.NULL).count() <= 1)
+             */.filter(types -> types.stream().filter(t -> t == DataType.NULL).count() <= 1)
             .map(types -> typeErrorSupplier(validPerPosition.size() != 1, validPerPosition, types, typeErrorMessageSupplier))
             .forEach(suppliers::add);
         return suppliers;
@@ -760,13 +759,13 @@ public static String errorMessageStringForBinaryOperators(
             return typeErrorMessage(includeOrdinal, validPerPosition, types);
         } catch (IllegalStateException e) {
             // This means all the positional args were okay, so the expected error is from the combination
-            if (types.get(0).equals(DataTypes.UNSIGNED_LONG)) {
+            if (types.get(0).equals(DataType.UNSIGNED_LONG)) {
                 return "first argument of [] is [unsigned_long] and second is ["
                     + types.get(1).typeName()
                     + "]. [unsigned_long] can only be operated on together with another [unsigned_long]";
 
             }
-            if (types.get(1).equals(DataTypes.UNSIGNED_LONG)) {
+            if (types.get(1).equals(DataType.UNSIGNED_LONG)) {
                 return "first argument of [] is ["
                     + types.get(0).typeName()
                     + "] and second is [unsigned_long]. [unsigned_long] can only be operated on together with another [unsigned_long]";
@@ -908,186 +907,186 @@ protected static String typeErrorMessage(boolean includeOrdinal, List<Set<DataTy
 
     private static final Map<Set<DataType>, String> NAMED_EXPECTED_TYPES = Map.ofEntries(
         Map.entry(
-            Set.of(DataTypes.DATE_PERIOD, DataTypes.DOUBLE, DataTypes.INTEGER, DataTypes.LONG, DataTypes.TIME_DURATION, DataTypes.NULL),
+            Set.of(DataType.DATE_PERIOD, DataType.DOUBLE, DataType.INTEGER, DataType.LONG, DataType.TIME_DURATION, DataType.NULL),
             "numeric, date_period or time_duration"
         ),
-        Map.entry(Set.of(DataTypes.DATETIME, DataTypes.NULL), "datetime"),
-        Map.entry(Set.of(DataTypes.DOUBLE, DataTypes.NULL), "double"),
-        Map.entry(Set.of(DataTypes.INTEGER, DataTypes.NULL), "integer"),
-        Map.entry(Set.of(DataTypes.IP, DataTypes.NULL), "ip"),
-        Map.entry(Set.of(DataTypes.LONG, DataTypes.INTEGER, DataTypes.UNSIGNED_LONG, DataTypes.DOUBLE, DataTypes.NULL), "numeric"),
-        Map.entry(Set.of(DataTypes.LONG, DataTypes.INTEGER, DataTypes.UNSIGNED_LONG, DataTypes.DOUBLE), "numeric"),
-        Map.entry(Set.of(DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.VERSION, DataTypes.NULL), "string or version"),
-        Map.entry(Set.of(DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL), "string"),
-        Map.entry(Set.of(DataTypes.IP, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL), "ip or string"),
+        Map.entry(Set.of(DataType.DATETIME, DataType.NULL), "datetime"),
+        Map.entry(Set.of(DataType.DOUBLE, DataType.NULL), "double"),
+        Map.entry(Set.of(DataType.INTEGER, DataType.NULL), "integer"),
+        Map.entry(Set.of(DataType.IP, DataType.NULL), "ip"),
+        Map.entry(Set.of(DataType.LONG, DataType.INTEGER, DataType.UNSIGNED_LONG, DataType.DOUBLE, DataType.NULL), "numeric"),
+        Map.entry(Set.of(DataType.LONG, DataType.INTEGER, DataType.UNSIGNED_LONG, DataType.DOUBLE), "numeric"),
+        Map.entry(Set.of(DataType.KEYWORD, DataType.TEXT, DataType.VERSION, DataType.NULL), "string or version"),
+        Map.entry(Set.of(DataType.KEYWORD, DataType.TEXT, DataType.NULL), "string"),
+        Map.entry(Set.of(DataType.IP, DataType.KEYWORD, DataType.TEXT, DataType.NULL), "ip or string"),
         Map.entry(Set.copyOf(Arrays.asList(representableTypes())), "representable"),
         Map.entry(Set.copyOf(Arrays.asList(representableNonSpatialTypes())), "representableNonSpatial"),
         Map.entry(
             Set.of(
-                DataTypes.BOOLEAN,
-                DataTypes.DOUBLE,
-                DataTypes.INTEGER,
-                DataTypes.KEYWORD,
-                DataTypes.LONG,
-                DataTypes.TEXT,
-                DataTypes.UNSIGNED_LONG,
-                DataTypes.NULL
+                DataType.BOOLEAN,
+                DataType.DOUBLE,
+                DataType.INTEGER,
+                DataType.KEYWORD,
+                DataType.LONG,
+                DataType.TEXT,
+                DataType.UNSIGNED_LONG,
+                DataType.NULL
             ),
             "boolean or numeric or string"
         ),
         Map.entry(
             Set.of(
-                DataTypes.DATETIME,
-                DataTypes.DOUBLE,
-                DataTypes.INTEGER,
-                DataTypes.KEYWORD,
-                DataTypes.LONG,
-                DataTypes.TEXT,
-                DataTypes.UNSIGNED_LONG,
-                DataTypes.NULL
+                DataType.DATETIME,
+                DataType.DOUBLE,
+                DataType.INTEGER,
+                DataType.KEYWORD,
+                DataType.LONG,
+                DataType.TEXT,
+                DataType.UNSIGNED_LONG,
+                DataType.NULL
             ),
             "datetime or numeric or string"
         ),
         // What Add accepts
         Map.entry(
             Set.of(
-                DataTypes.DATE_PERIOD,
-                DataTypes.DATETIME,
-                DataTypes.DOUBLE,
-                DataTypes.INTEGER,
-                DataTypes.LONG,
-                DataTypes.NULL,
-                DataTypes.TIME_DURATION,
-                DataTypes.UNSIGNED_LONG
+                DataType.DATE_PERIOD,
+                DataType.DATETIME,
+                DataType.DOUBLE,
+                DataType.INTEGER,
+                DataType.LONG,
+                DataType.NULL,
+                DataType.TIME_DURATION,
+                DataType.UNSIGNED_LONG
             ),
             "datetime or numeric"
         ),
         Map.entry(
             Set.of(
-                DataTypes.BOOLEAN,
-                DataTypes.DATETIME,
-                DataTypes.DOUBLE,
-                DataTypes.INTEGER,
-                DataTypes.KEYWORD,
-                DataTypes.LONG,
-                DataTypes.TEXT,
-                DataTypes.UNSIGNED_LONG,
-                DataTypes.NULL
+                DataType.BOOLEAN,
+                DataType.DATETIME,
+                DataType.DOUBLE,
+                DataType.INTEGER,
+                DataType.KEYWORD,
+                DataType.LONG,
+                DataType.TEXT,
+                DataType.UNSIGNED_LONG,
+                DataType.NULL
             ),
             "boolean or datetime or numeric or string"
         ),
         // to_int
         Map.entry(
             Set.of(
-                DataTypes.BOOLEAN,
-                DataTypes.COUNTER_INTEGER,
-                DataTypes.DATETIME,
-                DataTypes.DOUBLE,
-                DataTypes.INTEGER,
-                DataTypes.KEYWORD,
-                DataTypes.LONG,
-                DataTypes.TEXT,
-                DataTypes.UNSIGNED_LONG,
-                DataTypes.NULL
+                DataType.BOOLEAN,
+                DataType.COUNTER_INTEGER,
+                DataType.DATETIME,
+                DataType.DOUBLE,
+                DataType.INTEGER,
+                DataType.KEYWORD,
+                DataType.LONG,
+                DataType.TEXT,
+                DataType.UNSIGNED_LONG,
+                DataType.NULL
             ),
             "boolean or counter_integer or datetime or numeric or string"
         ),
         // to_long
         Map.entry(
             Set.of(
-                DataTypes.BOOLEAN,
-                DataTypes.COUNTER_INTEGER,
-                DataTypes.COUNTER_LONG,
-                DataTypes.DATETIME,
-                DataTypes.DOUBLE,
-                DataTypes.INTEGER,
-                DataTypes.KEYWORD,
-                DataTypes.LONG,
-                DataTypes.TEXT,
-                DataTypes.UNSIGNED_LONG,
-                DataTypes.NULL
+                DataType.BOOLEAN,
+                DataType.COUNTER_INTEGER,
+                DataType.COUNTER_LONG,
+                DataType.DATETIME,
+                DataType.DOUBLE,
+                DataType.INTEGER,
+                DataType.KEYWORD,
+                DataType.LONG,
+                DataType.TEXT,
+                DataType.UNSIGNED_LONG,
+                DataType.NULL
             ),
             "boolean or counter_integer or counter_long or datetime or numeric or string"
         ),
         // to_double
         Map.entry(
             Set.of(
-                DataTypes.BOOLEAN,
-                DataTypes.COUNTER_DOUBLE,
-                DataTypes.COUNTER_INTEGER,
-                DataTypes.COUNTER_LONG,
-                DataTypes.DATETIME,
-                DataTypes.DOUBLE,
-                DataTypes.INTEGER,
-                DataTypes.KEYWORD,
-                DataTypes.LONG,
-                DataTypes.TEXT,
-                DataTypes.UNSIGNED_LONG,
-                DataTypes.NULL
+                DataType.BOOLEAN,
+                DataType.COUNTER_DOUBLE,
+                DataType.COUNTER_INTEGER,
+                DataType.COUNTER_LONG,
+                DataType.DATETIME,
+                DataType.DOUBLE,
+                DataType.INTEGER,
+                DataType.KEYWORD,
+                DataType.LONG,
+                DataType.TEXT,
+                DataType.UNSIGNED_LONG,
+                DataType.NULL
             ),
             "boolean or counter_double or counter_integer or counter_long or datetime or numeric or string"
         ),
         Map.entry(
             Set.of(
-                DataTypes.BOOLEAN,
-                DataTypes.CARTESIAN_POINT,
-                DataTypes.DATETIME,
-                DataTypes.DOUBLE,
-                DataTypes.GEO_POINT,
-                DataTypes.INTEGER,
-                DataTypes.KEYWORD,
-                DataTypes.LONG,
-                DataTypes.TEXT,
-                DataTypes.UNSIGNED_LONG,
-                DataTypes.NULL
+                DataType.BOOLEAN,
+                DataType.CARTESIAN_POINT,
+                DataType.DATETIME,
+                DataType.DOUBLE,
+                DataType.GEO_POINT,
+                DataType.INTEGER,
+                DataType.KEYWORD,
+                DataType.LONG,
+                DataType.TEXT,
+                DataType.UNSIGNED_LONG,
+                DataType.NULL
             ),
             "boolean or cartesian_point or datetime or geo_point or numeric or string"
         ),
         Map.entry(
             Set.of(
-                DataTypes.DATETIME,
-                DataTypes.DOUBLE,
-                DataTypes.INTEGER,
-                DataTypes.IP,
-                DataTypes.KEYWORD,
-                DataTypes.LONG,
-                DataTypes.TEXT,
-                DataTypes.UNSIGNED_LONG,
-                DataTypes.VERSION,
-                DataTypes.NULL
+                DataType.DATETIME,
+                DataType.DOUBLE,
+                DataType.INTEGER,
+                DataType.IP,
+                DataType.KEYWORD,
+                DataType.LONG,
+                DataType.TEXT,
+                DataType.UNSIGNED_LONG,
+                DataType.VERSION,
+                DataType.NULL
             ),
             "datetime, double, integer, ip, keyword, long, text, unsigned_long or version"
         ),
         Map.entry(
             Set.of(
-                DataTypes.BOOLEAN,
-                DataTypes.DATETIME,
-                DataTypes.DOUBLE,
-                DataTypes.GEO_POINT,
-                DataTypes.GEO_SHAPE,
-                DataTypes.INTEGER,
-                DataTypes.IP,
-                DataTypes.KEYWORD,
-                DataTypes.LONG,
-                DataTypes.TEXT,
-                DataTypes.UNSIGNED_LONG,
-                DataTypes.VERSION,
-                DataTypes.NULL
+                DataType.BOOLEAN,
+                DataType.DATETIME,
+                DataType.DOUBLE,
+                DataType.GEO_POINT,
+                DataType.GEO_SHAPE,
+                DataType.INTEGER,
+                DataType.IP,
+                DataType.KEYWORD,
+                DataType.LONG,
+                DataType.TEXT,
+                DataType.UNSIGNED_LONG,
+                DataType.VERSION,
+                DataType.NULL
             ),
             "cartesian_point or datetime or geo_point or numeric or string"
         ),
-        Map.entry(Set.of(DataTypes.GEO_POINT, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL), "geo_point or string"),
-        Map.entry(Set.of(DataTypes.CARTESIAN_POINT, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL), "cartesian_point or string"),
+        Map.entry(Set.of(DataType.GEO_POINT, DataType.KEYWORD, DataType.TEXT, DataType.NULL), "geo_point or string"),
+        Map.entry(Set.of(DataType.CARTESIAN_POINT, DataType.KEYWORD, DataType.TEXT, DataType.NULL), "cartesian_point or string"),
         Map.entry(
-            Set.of(DataTypes.GEO_POINT, DataTypes.GEO_SHAPE, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL),
+            Set.of(DataType.GEO_POINT, DataType.GEO_SHAPE, DataType.KEYWORD, DataType.TEXT, DataType.NULL),
             "geo_point or geo_shape or string"
         ),
         Map.entry(
-            Set.of(DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE, DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.NULL),
+            Set.of(DataType.CARTESIAN_POINT, DataType.CARTESIAN_SHAPE, DataType.KEYWORD, DataType.TEXT, DataType.NULL),
             "cartesian_point or cartesian_shape or string"
         ),
-        Map.entry(Set.of(DataTypes.GEO_POINT, DataTypes.CARTESIAN_POINT, DataTypes.NULL), "geo_point or cartesian_point"),
-        Map.entry(Set.of(DataTypes.DATE_PERIOD, DataTypes.TIME_DURATION, DataTypes.NULL), "dateperiod or timeduration")
+        Map.entry(Set.of(DataType.GEO_POINT, DataType.CARTESIAN_POINT, DataType.NULL), "geo_point or cartesian_point"),
+        Map.entry(Set.of(DataType.DATE_PERIOD, DataType.TIME_DURATION, DataType.NULL), "dateperiod or timeduration")
     );
 
     // TODO: generate this message dynamically, a la AbstractConvertFunction#supportedTypesNames()?
@@ -1107,7 +1106,7 @@ private static String expectedType(Set<DataType> validTypes) {
     }
 
     protected static Stream<DataType> representable() {
-        return DataTypes.types().stream().filter(EsqlDataTypes::isRepresentable);
+        return DataType.types().stream().filter(EsqlDataTypes::isRepresentable);
     }
 
     protected static DataType[] representableTypes() {
@@ -1185,7 +1184,7 @@ private static Map<List<DataType>, DataType> signatures() {
             if (tc.getExpectedTypeError() != null) {
                 continue;
             }
-            if (tc.getData().stream().anyMatch(t -> t.type() == DataTypes.NULL)) {
+            if (tc.getData().stream().anyMatch(t -> t.type() == DataType.NULL)) {
                 continue;
             }
             signatures.putIfAbsent(tc.getData().stream().map(TestCaseSupplier.TypedData::type).toList(), tc.expectedType());
@@ -1602,6 +1601,6 @@ static Version randomVersion() {
      * All string types (keyword, text, match_only_text, etc).
      */
     protected static DataType[] strings() {
-        return DataTypes.types().stream().filter(DataTypes::isString).toArray(DataType[]::new);
+        return DataType.types().stream().filter(DataType::isString).toArray(DataType[]::new);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
index ee542232aa30b..7ff87a682a789 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/FieldAttributeTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.AbstractEsFieldTypeTests;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 
 public class FieldAttributeTests extends AbstractAttributeTestCase<FieldAttribute> {
@@ -21,7 +20,7 @@ static FieldAttribute createFieldAttribute(int maxDepth) {
         Source source = Source.EMPTY;
         FieldAttribute parent = maxDepth == 0 || randomBoolean() ? null : createFieldAttribute(maxDepth - 1);
         String name = randomAlphaOfLength(5);
-        DataType type = randomFrom(DataTypes.types());
+        DataType type = randomFrom(DataType.types());
         EsField field = AbstractEsFieldTypeTests.randomAnyEsField(maxDepth);
         String qualifier = randomBoolean() ? null : randomAlphaOfLength(3);
         Nullability nullability = randomFrom(Nullability.values());
@@ -47,7 +46,7 @@ protected FieldAttribute mutate(FieldAttribute instance) {
         switch (between(0, 6)) {
             case 0 -> parent = randomValueOtherThan(parent, () -> randomBoolean() ? null : createFieldAttribute(2));
             case 1 -> name = randomAlphaOfLength(name.length() + 1);
-            case 2 -> type = randomValueOtherThan(type, () -> randomFrom(DataTypes.types()));
+            case 2 -> type = randomValueOtherThan(type, () -> randomFrom(DataType.types()));
             case 3 -> field = randomValueOtherThan(field, () -> AbstractEsFieldTypeTests.randomAnyEsField(3));
             case 4 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
             case 5 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java
index 16a83b42d10ab..573af9c17bb1d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MetadataAttributeTests.java
@@ -12,14 +12,13 @@
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 public class MetadataAttributeTests extends AbstractAttributeTestCase<MetadataAttribute> {
     @Override
     protected MetadataAttribute create() {
         Source source = Source.EMPTY;
         String name = randomAlphaOfLength(5);
-        DataType type = randomFrom(DataTypes.types());
+        DataType type = randomFrom(DataType.types());
         String qualifier = randomBoolean() ? null : randomAlphaOfLength(3);
         Nullability nullability = randomFrom(Nullability.values());
         boolean synthetic = randomBoolean();
@@ -38,7 +37,7 @@ protected MetadataAttribute mutate(MetadataAttribute instance) {
         boolean searchable = instance.searchable();
         switch (between(0, 5)) {
             case 0 -> name = randomAlphaOfLength(name.length() + 1);
-            case 1 -> type = randomValueOtherThan(type, () -> randomFrom(DataTypes.types()));
+            case 1 -> type = randomValueOtherThan(type, () -> randomFrom(DataType.types()));
             case 2 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
             case 3 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
             case 4 -> synthetic = false == synthetic;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
index e248b741ff48d..716d4fa1f5cce 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
@@ -12,14 +12,13 @@
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 
 public class ReferenceAttributeTests extends AbstractAttributeTestCase<ReferenceAttribute> {
     @Override
     protected ReferenceAttribute create() {
         Source source = Source.EMPTY;
         String name = randomAlphaOfLength(5);
-        DataType type = randomFrom(DataTypes.types());
+        DataType type = randomFrom(DataType.types());
         String qualifier = randomBoolean() ? null : randomAlphaOfLength(3);
         Nullability nullability = randomFrom(Nullability.values());
         boolean synthetic = randomBoolean();
@@ -36,7 +35,7 @@ protected ReferenceAttribute mutate(ReferenceAttribute instance) {
         boolean synthetic = instance.synthetic();
         switch (between(0, 4)) {
             case 0 -> name = randomAlphaOfLength(name.length() + 1);
-            case 1 -> type = randomValueOtherThan(type, () -> randomFrom(DataTypes.types()));
+            case 1 -> type = randomValueOtherThan(type, () -> randomFrom(DataType.types()));
             case 2 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
             case 3 -> nullability = randomValueOtherThan(nullability, () -> randomFrom(Nullability.values()));
             case 4 -> synthetic = false == synthetic;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
index 9b86482a1ec88..54c4f2ae07eca 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -124,7 +123,7 @@ public String toString() {
 
     /**
      * Generate positive test cases for unary functions that operate on an {@code numeric}
-     * fields by casting them to {@link DataTypes#DOUBLE}s.
+     * fields by casting them to {@link DataType#DOUBLE}s.
      */
     public static List<TestCaseSupplier> forUnaryCastingToDouble(
         String name,
@@ -139,8 +138,8 @@ public static List<TestCaseSupplier> forUnaryCastingToDouble(
         List<TestCaseSupplier> suppliers = new ArrayList<>();
         forUnaryInt(
             suppliers,
-            eval + castToDoubleEvaluator(read, DataTypes.INTEGER) + "]",
-            DataTypes.DOUBLE,
+            eval + castToDoubleEvaluator(read, DataType.INTEGER) + "]",
+            DataType.DOUBLE,
             i -> expected.apply(Double.valueOf(i)),
             min.intValue(),
             max.intValue(),
@@ -148,8 +147,8 @@ public static List<TestCaseSupplier> forUnaryCastingToDouble(
         );
         forUnaryLong(
             suppliers,
-            eval + castToDoubleEvaluator(read, DataTypes.LONG) + "]",
-            DataTypes.DOUBLE,
+            eval + castToDoubleEvaluator(read, DataType.LONG) + "]",
+            DataType.DOUBLE,
             i -> expected.apply(Double.valueOf(i)),
             min.longValue(),
             max.longValue(),
@@ -157,20 +156,20 @@ public static List<TestCaseSupplier> forUnaryCastingToDouble(
         );
         forUnaryUnsignedLong(
             suppliers,
-            eval + castToDoubleEvaluator(read, DataTypes.UNSIGNED_LONG) + "]",
-            DataTypes.DOUBLE,
+            eval + castToDoubleEvaluator(read, DataType.UNSIGNED_LONG) + "]",
+            DataType.DOUBLE,
             ul -> expected.apply(ul.doubleValue()),
             BigInteger.valueOf((int) Math.ceil(min)),
             BigInteger.valueOf((int) Math.floor(max)),
             warnings
         );
-        forUnaryDouble(suppliers, eval + read + "]", DataTypes.DOUBLE, expected::apply, min, max, warnings);
+        forUnaryDouble(suppliers, eval + read + "]", DataType.DOUBLE, expected::apply, min, max, warnings);
         return suppliers;
     }
 
     /**
      * Generate positive test cases for binary functions that operate on an {@code numeric}
-     * fields by casting them to {@link DataTypes#DOUBLE}s.
+     * fields by casting them to {@link DataType#DOUBLE}s.
      */
     public static List<TestCaseSupplier> forBinaryCastingToDouble(
         String name,
@@ -216,7 +215,7 @@ public static List<TestCaseSupplier> forBinaryCastingToDouble(
             ),
             (lhs, rhs) -> warnings,
             suppliers,
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             false
         );
         return suppliers;
@@ -294,13 +293,13 @@ public record NumericTypeTestConfigs<T>(
         NumericTypeTestConfig<T> doubleStuff
     ) {
         public NumericTypeTestConfig<T> get(DataType type) {
-            if (type == DataTypes.INTEGER) {
+            if (type == DataType.INTEGER) {
                 return intStuff;
             }
-            if (type == DataTypes.LONG) {
+            if (type == DataType.LONG) {
                 return longStuff;
             }
-            if (type == DataTypes.DOUBLE) {
+            if (type == DataType.DOUBLE) {
                 return doubleStuff;
             }
             throw new IllegalArgumentException("bogus numeric type [" + type + "]");
@@ -311,30 +310,30 @@ public static DataType widen(DataType lhs, DataType rhs) {
         if (lhs == rhs) {
             return lhs;
         }
-        if (lhs == DataTypes.DOUBLE || rhs == DataTypes.DOUBLE) {
-            return DataTypes.DOUBLE;
+        if (lhs == DataType.DOUBLE || rhs == DataType.DOUBLE) {
+            return DataType.DOUBLE;
         }
-        if (lhs == DataTypes.LONG || rhs == DataTypes.LONG) {
-            return DataTypes.LONG;
+        if (lhs == DataType.LONG || rhs == DataType.LONG) {
+            return DataType.LONG;
         }
         throw new IllegalArgumentException("Invalid numeric widening lhs: [" + lhs + "] rhs: [" + rhs + "]");
     }
 
     public static List<TypedDataSupplier> getSuppliersForNumericType(DataType type, Number min, Number max, boolean includeZero) {
-        if (type == DataTypes.INTEGER) {
+        if (type == DataType.INTEGER) {
             return intCases(NumericUtils.saturatingIntValue(min), NumericUtils.saturatingIntValue(max), includeZero);
         }
-        if (type == DataTypes.LONG) {
+        if (type == DataType.LONG) {
             return longCases(min.longValue(), max.longValue(), includeZero);
         }
-        if (type == DataTypes.UNSIGNED_LONG) {
+        if (type == DataType.UNSIGNED_LONG) {
             return ulongCases(
                 min instanceof BigInteger ? (BigInteger) min : BigInteger.valueOf(Math.max(min.longValue(), 0L)),
                 max instanceof BigInteger ? (BigInteger) max : BigInteger.valueOf(Math.max(max.longValue(), 0L)),
                 includeZero
             );
         }
-        if (type == DataTypes.DOUBLE) {
+        if (type == DataType.DOUBLE) {
             return doubleCases(min.doubleValue(), max.doubleValue(), includeZero);
         }
         throw new IllegalArgumentException("bogus numeric type [" + type + "]");
@@ -348,7 +347,7 @@ public static List<TestCaseSupplier> forBinaryComparisonWithWidening(
         boolean allowRhsZero
     ) {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        List<DataType> numericTypes = List.of(DataTypes.INTEGER, DataTypes.LONG, DataTypes.DOUBLE);
+        List<DataType> numericTypes = List.of(DataType.INTEGER, DataType.LONG, DataType.DOUBLE);
 
         for (DataType lhsType : numericTypes) {
             for (DataType rhsType : numericTypes) {
@@ -371,7 +370,7 @@ public static List<TestCaseSupplier> forBinaryComparisonWithWidening(
                     (lhs, rhs) -> equalTo(evaluatorToString.apply(lhs, rhs)),
                     warnings,
                     suppliers,
-                    DataTypes.BOOLEAN,
+                    DataType.BOOLEAN,
                     true
                 );
             }
@@ -387,7 +386,7 @@ public static List<TestCaseSupplier> forBinaryWithWidening(
         boolean allowRhsZero
     ) {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        List<DataType> numericTypes = List.of(DataTypes.INTEGER, DataTypes.LONG, DataTypes.DOUBLE);
+        List<DataType> numericTypes = List.of(DataType.INTEGER, DataType.LONG, DataType.DOUBLE);
 
         for (DataType lhsType : numericTypes) {
             for (DataType rhsType : numericTypes) {
@@ -467,7 +466,7 @@ public static List<TestCaseSupplier> forBinaryNotCasting(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#INTEGER}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#INTEGER}.
      */
     public static void forUnaryInt(
         List<TestCaseSupplier> suppliers,
@@ -501,7 +500,7 @@ public static void forUnaryInt(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#LONG}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#LONG}.
      */
     public static void forUnaryLong(
         List<TestCaseSupplier> suppliers,
@@ -535,7 +534,7 @@ public static void forUnaryLong(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#UNSIGNED_LONG}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#UNSIGNED_LONG}.
      */
     public static void forUnaryUnsignedLong(
         List<TestCaseSupplier> suppliers,
@@ -569,7 +568,7 @@ public static void forUnaryUnsignedLong(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#DOUBLE}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#DOUBLE}.
      */
     public static void forUnaryDouble(
         List<TestCaseSupplier> suppliers,
@@ -603,7 +602,7 @@ public static void forUnaryDouble(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#BOOLEAN}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#BOOLEAN}.
      */
     public static void forUnaryBoolean(
         List<TestCaseSupplier> suppliers,
@@ -616,7 +615,7 @@ public static void forUnaryBoolean(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#DATETIME}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#DATETIME}.
      */
     public static void forUnaryDatetime(
         List<TestCaseSupplier> suppliers,
@@ -636,7 +635,7 @@ public static void forUnaryDatetime(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#GEO_POINT}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#GEO_POINT}.
      */
     public static void forUnaryGeoPoint(
         List<TestCaseSupplier> suppliers,
@@ -649,7 +648,7 @@ public static void forUnaryGeoPoint(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#CARTESIAN_POINT}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#CARTESIAN_POINT}.
      */
     public static void forUnaryCartesianPoint(
         List<TestCaseSupplier> suppliers,
@@ -662,7 +661,7 @@ public static void forUnaryCartesianPoint(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#GEO_SHAPE}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#GEO_SHAPE}.
      */
     public static void forUnaryGeoShape(
         List<TestCaseSupplier> suppliers,
@@ -675,7 +674,7 @@ public static void forUnaryGeoShape(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#CARTESIAN_SHAPE}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#CARTESIAN_SHAPE}.
      */
     public static void forUnaryCartesianShape(
         List<TestCaseSupplier> suppliers,
@@ -688,7 +687,7 @@ public static void forUnaryCartesianShape(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#IP}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#IP}.
      */
     public static void forUnaryIp(
         List<TestCaseSupplier> suppliers,
@@ -701,7 +700,7 @@ public static void forUnaryIp(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#KEYWORD} and {@link DataTypes#TEXT}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#KEYWORD} and {@link DataType#TEXT}.
      */
     public static void forUnaryStrings(
         List<TestCaseSupplier> suppliers,
@@ -733,7 +732,7 @@ public static void forUnaryStrings(
     }
 
     /**
-     * Generate positive test cases for a unary function operating on an {@link DataTypes#VERSION}.
+     * Generate positive test cases for a unary function operating on an {@link DataType#VERSION}.
      */
     public static void forUnaryVersion(
         List<TestCaseSupplier> suppliers,
@@ -824,23 +823,23 @@ public static void unary(
     public static List<TypedDataSupplier> intCases(int min, int max, boolean includeZero) {
         List<TypedDataSupplier> cases = new ArrayList<>();
         if (0 <= max && 0 >= min && includeZero) {
-            cases.add(new TypedDataSupplier("<0 int>", () -> 0, DataTypes.INTEGER));
+            cases.add(new TypedDataSupplier("<0 int>", () -> 0, DataType.INTEGER));
         }
 
         int lower = Math.max(min, 1);
         int upper = Math.min(max, Integer.MAX_VALUE);
         if (lower < upper) {
-            cases.add(new TypedDataSupplier("<positive int>", () -> ESTestCase.randomIntBetween(lower, upper), DataTypes.INTEGER));
+            cases.add(new TypedDataSupplier("<positive int>", () -> ESTestCase.randomIntBetween(lower, upper), DataType.INTEGER));
         } else if (lower == upper) {
-            cases.add(new TypedDataSupplier("<" + lower + " int>", () -> lower, DataTypes.INTEGER));
+            cases.add(new TypedDataSupplier("<" + lower + " int>", () -> lower, DataType.INTEGER));
         }
 
         int lower1 = Math.max(min, Integer.MIN_VALUE);
         int upper1 = Math.min(max, -1);
         if (lower1 < upper1) {
-            cases.add(new TypedDataSupplier("<negative int>", () -> ESTestCase.randomIntBetween(lower1, upper1), DataTypes.INTEGER));
+            cases.add(new TypedDataSupplier("<negative int>", () -> ESTestCase.randomIntBetween(lower1, upper1), DataType.INTEGER));
         } else if (lower1 == upper1) {
-            cases.add(new TypedDataSupplier("<" + lower1 + " int>", () -> lower1, DataTypes.INTEGER));
+            cases.add(new TypedDataSupplier("<" + lower1 + " int>", () -> lower1, DataType.INTEGER));
         }
         return cases;
     }
@@ -848,23 +847,23 @@ public static List<TypedDataSupplier> intCases(int min, int max, boolean include
     public static List<TypedDataSupplier> longCases(long min, long max, boolean includeZero) {
         List<TypedDataSupplier> cases = new ArrayList<>();
         if (0L <= max && 0L >= min && includeZero) {
-            cases.add(new TypedDataSupplier("<0 long>", () -> 0L, DataTypes.LONG));
+            cases.add(new TypedDataSupplier("<0 long>", () -> 0L, DataType.LONG));
         }
 
         long lower = Math.max(min, 1);
         long upper = Math.min(max, Long.MAX_VALUE);
         if (lower < upper) {
-            cases.add(new TypedDataSupplier("<positive long>", () -> ESTestCase.randomLongBetween(lower, upper), DataTypes.LONG));
+            cases.add(new TypedDataSupplier("<positive long>", () -> ESTestCase.randomLongBetween(lower, upper), DataType.LONG));
         } else if (lower == upper) {
-            cases.add(new TypedDataSupplier("<" + lower + " long>", () -> lower, DataTypes.LONG));
+            cases.add(new TypedDataSupplier("<" + lower + " long>", () -> lower, DataType.LONG));
         }
 
         long lower1 = Math.max(min, Long.MIN_VALUE);
         long upper1 = Math.min(max, -1);
         if (lower1 < upper1) {
-            cases.add(new TypedDataSupplier("<negative long>", () -> ESTestCase.randomLongBetween(lower1, upper1), DataTypes.LONG));
+            cases.add(new TypedDataSupplier("<negative long>", () -> ESTestCase.randomLongBetween(lower1, upper1), DataType.LONG));
         } else if (lower1 == upper1) {
-            cases.add(new TypedDataSupplier("<" + lower1 + " long>", () -> lower1, DataTypes.LONG));
+            cases.add(new TypedDataSupplier("<" + lower1 + " long>", () -> lower1, DataType.LONG));
         }
 
         return cases;
@@ -875,7 +874,7 @@ public static List<TypedDataSupplier> ulongCases(BigInteger min, BigInteger max,
 
         // Zero
         if (BigInteger.ZERO.compareTo(max) <= 0 && BigInteger.ZERO.compareTo(min) >= 0 && includeZero) {
-            cases.add(new TypedDataSupplier("<0 unsigned long>", () -> BigInteger.ZERO, DataTypes.UNSIGNED_LONG));
+            cases.add(new TypedDataSupplier("<0 unsigned long>", () -> BigInteger.ZERO, DataType.UNSIGNED_LONG));
         }
 
         // small values, less than Long.MAX_VALUE
@@ -886,11 +885,11 @@ public static List<TypedDataSupplier> ulongCases(BigInteger min, BigInteger max,
                 new TypedDataSupplier(
                     "<small unsigned long>",
                     () -> ESTestCase.randomUnsignedLongBetween(lower1, upper1),
-                    DataTypes.UNSIGNED_LONG
+                    DataType.UNSIGNED_LONG
                 )
             );
         } else if (lower1.compareTo(upper1) == 0) {
-            cases.add(new TypedDataSupplier("<small unsigned long>", () -> lower1, DataTypes.UNSIGNED_LONG));
+            cases.add(new TypedDataSupplier("<small unsigned long>", () -> lower1, DataType.UNSIGNED_LONG));
         }
 
         // Big values, greater than Long.MAX_VALUE
@@ -901,11 +900,11 @@ public static List<TypedDataSupplier> ulongCases(BigInteger min, BigInteger max,
                 new TypedDataSupplier(
                     "<big unsigned long>",
                     () -> ESTestCase.randomUnsignedLongBetween(lower2, upper2),
-                    DataTypes.UNSIGNED_LONG
+                    DataType.UNSIGNED_LONG
                 )
             );
         } else if (lower2.compareTo(upper2) == 0) {
-            cases.add(new TypedDataSupplier("<big unsigned long>", () -> lower2, DataTypes.UNSIGNED_LONG));
+            cases.add(new TypedDataSupplier("<big unsigned long>", () -> lower2, DataType.UNSIGNED_LONG));
         }
         return cases;
     }
@@ -915,8 +914,8 @@ public static List<TypedDataSupplier> doubleCases(double min, double max, boolea
 
         // Zeros
         if (0d <= max && 0d >= min && includeZero) {
-            cases.add(new TypedDataSupplier("<0 double>", () -> 0.0d, DataTypes.DOUBLE));
-            cases.add(new TypedDataSupplier("<-0 double>", () -> -0.0d, DataTypes.DOUBLE));
+            cases.add(new TypedDataSupplier("<0 double>", () -> 0.0d, DataType.DOUBLE));
+            cases.add(new TypedDataSupplier("<-0 double>", () -> -0.0d, DataType.DOUBLE));
         }
 
         // Positive small double
@@ -927,11 +926,11 @@ public static List<TypedDataSupplier> doubleCases(double min, double max, boolea
                 new TypedDataSupplier(
                     "<small positive double>",
                     () -> ESTestCase.randomDoubleBetween(lower1, upper1, true),
-                    DataTypes.DOUBLE
+                    DataType.DOUBLE
                 )
             );
         } else if (lower1 == upper1) {
-            cases.add(new TypedDataSupplier("<small positive double>", () -> lower1, DataTypes.DOUBLE));
+            cases.add(new TypedDataSupplier("<small positive double>", () -> lower1, DataType.DOUBLE));
         }
 
         // Negative small double
@@ -942,11 +941,11 @@ public static List<TypedDataSupplier> doubleCases(double min, double max, boolea
                 new TypedDataSupplier(
                     "<small negative double>",
                     () -> ESTestCase.randomDoubleBetween(lower2, upper2, true),
-                    DataTypes.DOUBLE
+                    DataType.DOUBLE
                 )
             );
         } else if (lower2 == upper2) {
-            cases.add(new TypedDataSupplier("<small negative double>", () -> lower2, DataTypes.DOUBLE));
+            cases.add(new TypedDataSupplier("<small negative double>", () -> lower2, DataType.DOUBLE));
         }
 
         // Positive big double
@@ -954,10 +953,10 @@ public static List<TypedDataSupplier> doubleCases(double min, double max, boolea
         double upper3 = Math.min(Double.MAX_VALUE, max);
         if (lower3 < upper3) {
             cases.add(
-                new TypedDataSupplier("<big positive double>", () -> ESTestCase.randomDoubleBetween(lower3, upper3, true), DataTypes.DOUBLE)
+                new TypedDataSupplier("<big positive double>", () -> ESTestCase.randomDoubleBetween(lower3, upper3, true), DataType.DOUBLE)
             );
         } else if (lower3 == upper3) {
-            cases.add(new TypedDataSupplier("<big positive double>", () -> lower3, DataTypes.DOUBLE));
+            cases.add(new TypedDataSupplier("<big positive double>", () -> lower3, DataType.DOUBLE));
         }
 
         // Negative big double
@@ -966,47 +965,47 @@ public static List<TypedDataSupplier> doubleCases(double min, double max, boolea
         double upper4 = Math.min(-1, max); // because again, the interval from -1 to 0 is very high density
         if (lower4 < upper4) {
             cases.add(
-                new TypedDataSupplier("<big negative double>", () -> ESTestCase.randomDoubleBetween(lower4, upper4, true), DataTypes.DOUBLE)
+                new TypedDataSupplier("<big negative double>", () -> ESTestCase.randomDoubleBetween(lower4, upper4, true), DataType.DOUBLE)
             );
         } else if (lower4 == upper4) {
-            cases.add(new TypedDataSupplier("<big negative double>", () -> lower4, DataTypes.DOUBLE));
+            cases.add(new TypedDataSupplier("<big negative double>", () -> lower4, DataType.DOUBLE));
         }
         return cases;
     }
 
     public static List<TypedDataSupplier> booleanCases() {
         return List.of(
-            new TypedDataSupplier("<true>", () -> true, DataTypes.BOOLEAN),
-            new TypedDataSupplier("<false>", () -> false, DataTypes.BOOLEAN)
+            new TypedDataSupplier("<true>", () -> true, DataType.BOOLEAN),
+            new TypedDataSupplier("<false>", () -> false, DataType.BOOLEAN)
         );
     }
 
     public static List<TypedDataSupplier> dateCases() {
         return List.of(
-            new TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataTypes.DATETIME),
+            new TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataType.DATETIME),
             new TypedDataSupplier(
                 "<date>",
                 () -> ESTestCase.randomLongBetween(0, 10 * (long) 10e11), // 1970-01-01T00:00:00Z - 2286-11-20T17:46:40Z
-                DataTypes.DATETIME
+                DataType.DATETIME
             ),
             new TypedDataSupplier(
                 "<far future date>",
                 // 2286-11-20T17:46:40Z - +292278994-08-17T07:12:55.807Z
                 () -> ESTestCase.randomLongBetween(10 * (long) 10e11, Long.MAX_VALUE),
-                DataTypes.DATETIME
+                DataType.DATETIME
             ),
             new TypedDataSupplier(
                 "<near the end of time>",
                 // very close to +292278994-08-17T07:12:55.807Z, the maximum supported millis since epoch
                 () -> ESTestCase.randomLongBetween(Long.MAX_VALUE / 100 * 99, Long.MAX_VALUE),
-                DataTypes.DATETIME
+                DataType.DATETIME
             )
         );
     }
 
     public static List<TypedDataSupplier> datePeriodCases() {
         return List.of(
-            new TypedDataSupplier("<zero date period>", () -> Period.ZERO, DataTypes.DATE_PERIOD, true),
+            new TypedDataSupplier("<zero date period>", () -> Period.ZERO, DataType.DATE_PERIOD, true),
             new TypedDataSupplier(
                 "<random date period>",
                 () -> Period.of(
@@ -1014,7 +1013,7 @@ public static List<TypedDataSupplier> datePeriodCases() {
                     ESTestCase.randomIntBetween(-13, 13),
                     ESTestCase.randomIntBetween(-32, 32)
                 ),
-                DataTypes.DATE_PERIOD,
+                DataType.DATE_PERIOD,
                 true
             )
         );
@@ -1022,11 +1021,11 @@ public static List<TypedDataSupplier> datePeriodCases() {
 
     public static List<TypedDataSupplier> timeDurationCases() {
         return List.of(
-            new TypedDataSupplier("<zero time duration>", () -> Duration.ZERO, DataTypes.TIME_DURATION, true),
+            new TypedDataSupplier("<zero time duration>", () -> Duration.ZERO, DataType.TIME_DURATION, true),
             new TypedDataSupplier(
                 "<up to 7 days duration>",
                 () -> Duration.ofMillis(ESTestCase.randomLongBetween(-604800000L, 604800000L)), // plus/minus 7 days
-                DataTypes.TIME_DURATION,
+                DataType.TIME_DURATION,
                 true
             )
         );
@@ -1050,7 +1049,7 @@ public static List<TypedDataSupplier> cartesianShapeCases() {
 
     public static List<TypedDataSupplier> geoPointCases(Supplier<Boolean> hasAlt) {
         return List.of(
-            new TypedDataSupplier("<geo_point>", () -> GEO.asWkb(GeometryTestUtils.randomPoint(hasAlt.get())), DataTypes.GEO_POINT)
+            new TypedDataSupplier("<geo_point>", () -> GEO.asWkb(GeometryTestUtils.randomPoint(hasAlt.get())), DataType.GEO_POINT)
         );
     }
 
@@ -1059,7 +1058,7 @@ public static List<TypedDataSupplier> cartesianPointCases(Supplier<Boolean> hasA
             new TypedDataSupplier(
                 "<cartesian_point>",
                 () -> CARTESIAN.asWkb(ShapeTestUtils.randomPoint(hasAlt.get())),
-                DataTypes.CARTESIAN_POINT
+                DataType.CARTESIAN_POINT
             )
         );
     }
@@ -1069,7 +1068,7 @@ public static List<TypedDataSupplier> geoShapeCases(Supplier<Boolean> hasAlt) {
             new TypedDataSupplier(
                 "<geo_shape>",
                 () -> GEO.asWkb(GeometryTestUtils.randomGeometryWithoutCircle(0, hasAlt.get())),
-                DataTypes.GEO_SHAPE
+                DataType.GEO_SHAPE
             )
         );
     }
@@ -1079,7 +1078,7 @@ public static List<TypedDataSupplier> cartesianShapeCases(Supplier<Boolean> hasA
             new TypedDataSupplier(
                 "<cartesian_shape>",
                 () -> CARTESIAN.asWkb(ShapeTestUtils.randomGeometry(hasAlt.get())),
-                DataTypes.CARTESIAN_SHAPE
+                DataType.CARTESIAN_SHAPE
             )
         );
     }
@@ -1089,10 +1088,10 @@ public static List<TypedDataSupplier> ipCases() {
             new TypedDataSupplier(
                 "<127.0.0.1 ip>",
                 () -> new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1"))),
-                DataTypes.IP
+                DataType.IP
             ),
-            new TypedDataSupplier("<ipv4>", () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(true))), DataTypes.IP),
-            new TypedDataSupplier("<ipv6>", () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(false))), DataTypes.IP)
+            new TypedDataSupplier("<ipv4>", () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(true))), DataType.IP),
+            new TypedDataSupplier("<ipv6>", () -> new BytesRef(InetAddressPoint.encode(ESTestCase.randomIp(false))), DataType.IP)
         );
     }
 
@@ -1130,18 +1129,18 @@ public static List<TypedDataSupplier> versionCases(String prefix) {
             new TypedDataSupplier(
                 "<" + prefix + "version major>",
                 () -> new Version(Integer.toString(ESTestCase.between(0, 100))).toBytesRef(),
-                DataTypes.VERSION
+                DataType.VERSION
             ),
             new TypedDataSupplier(
                 "<" + prefix + "version major.minor>",
                 () -> new Version(ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100)).toBytesRef(),
-                DataTypes.VERSION
+                DataType.VERSION
             ),
             new TypedDataSupplier(
                 "<" + prefix + "version major.minor.patch>",
                 () -> new Version(ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100) + "." + ESTestCase.between(0, 100))
                     .toBytesRef(),
-                DataTypes.VERSION
+                DataType.VERSION
             )
         );
     }
@@ -1150,61 +1149,61 @@ public static String getCastEvaluator(String original, DataType current, DataTyp
         if (current == target) {
             return original;
         }
-        if (target == DataTypes.LONG) {
+        if (target == DataType.LONG) {
             return castToLongEvaluator(original, current);
         }
-        if (target == DataTypes.UNSIGNED_LONG) {
+        if (target == DataType.UNSIGNED_LONG) {
             return castToUnsignedLongEvaluator(original, current);
         }
-        if (target == DataTypes.DOUBLE) {
+        if (target == DataType.DOUBLE) {
             return castToDoubleEvaluator(original, current);
         }
         throw new IllegalArgumentException("Invalid numeric cast to [" + target + "]");
     }
 
     private static String castToLongEvaluator(String original, DataType current) {
-        if (current == DataTypes.LONG) {
+        if (current == DataType.LONG) {
             return original;
         }
-        if (current == DataTypes.INTEGER) {
+        if (current == DataType.INTEGER) {
             return "CastIntToLongEvaluator[v=" + original + "]";
         }
-        if (current == DataTypes.DOUBLE) {
+        if (current == DataType.DOUBLE) {
             return "CastDoubleToLongEvaluator[v=" + original + "]";
         }
-        if (current == DataTypes.UNSIGNED_LONG) {
+        if (current == DataType.UNSIGNED_LONG) {
             return "CastUnsignedLongToLong[v=" + original + "]";
         }
         throw new UnsupportedOperationException();
     }
 
     private static String castToUnsignedLongEvaluator(String original, DataType current) {
-        if (current == DataTypes.UNSIGNED_LONG) {
+        if (current == DataType.UNSIGNED_LONG) {
             return original;
         }
-        if (current == DataTypes.INTEGER) {
+        if (current == DataType.INTEGER) {
             return "CastIntToUnsignedLongEvaluator[v=" + original + "]";
         }
-        if (current == DataTypes.LONG) {
+        if (current == DataType.LONG) {
             return "CastLongToUnsignedLongEvaluator[v=" + original + "]";
         }
-        if (current == DataTypes.DOUBLE) {
+        if (current == DataType.DOUBLE) {
             return "CastDoubleToUnsignedLongEvaluator[v=" + original + "]";
         }
         throw new UnsupportedOperationException();
     }
 
     private static String castToDoubleEvaluator(String original, DataType current) {
-        if (current == DataTypes.DOUBLE) {
+        if (current == DataType.DOUBLE) {
             return original;
         }
-        if (current == DataTypes.INTEGER) {
+        if (current == DataType.INTEGER) {
             return "CastIntToDoubleEvaluator[v=" + original + "]";
         }
-        if (current == DataTypes.LONG) {
+        if (current == DataType.LONG) {
             return "CastLongToDoubleEvaluator[v=" + original + "]";
         }
-        if (current == DataTypes.UNSIGNED_LONG) {
+        if (current == DataType.UNSIGNED_LONG) {
             return "CastUnsignedLongToDoubleEvaluator[v=" + original + "]";
         }
         throw new UnsupportedOperationException();
@@ -1379,7 +1378,7 @@ public TypedData get() {
      * Holds a data value and the intended parse type of that value
      */
     public static class TypedData {
-        public static final TypedData NULL = new TypedData(null, DataTypes.NULL, "<null>");
+        public static final TypedData NULL = new TypedData(null, DataType.NULL, "<null>");
 
         private final Object data;
         private final DataType type;
@@ -1393,7 +1392,7 @@ public static class TypedData {
          * @param forceLiteral should this data always be converted to a literal and <strong>never</strong> to a field reference?
          */
         private TypedData(Object data, DataType type, String name, boolean forceLiteral) {
-            if (type == DataTypes.UNSIGNED_LONG && data instanceof BigInteger b) {
+            if (type == DataType.UNSIGNED_LONG && data instanceof BigInteger b) {
                 this.data = NumericUtils.asLongUnsigned(b);
             } else {
                 this.data = data;
@@ -1447,7 +1446,7 @@ public TypedData forceValueToNull() {
 
         @Override
         public String toString() {
-            if (type == DataTypes.UNSIGNED_LONG && data instanceof Long longData) {
+            if (type == DataType.UNSIGNED_LONG && data instanceof Long longData) {
                 return type.toString() + "(" + NumericUtils.unsignedLongAsBigInteger(longData).toString() + ")";
             }
             return type.toString() + "(" + (data == null ? "null" : data.toString()) + ")";
@@ -1491,7 +1490,7 @@ public Object data() {
          * @return the data value being supplied, casting unsigned longs into BigIntegers correctly
          */
         public Object getValue() {
-            if (type == DataTypes.UNSIGNED_LONG && data instanceof Long l) {
+            if (type == DataType.UNSIGNED_LONG && data instanceof Long l) {
                 return NumericUtils.unsignedLongAsBigInteger(l);
             }
             return data;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java
index 18957a7f115b8..c8fe9e536beea 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.tree.Location;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.hamcrest.Matcher;
@@ -63,15 +62,15 @@ protected final ArgumentSpec optional(DataType... validTypes) {
 
     private Set<DataType> withNullAndSorted(DataType[] validTypes) {
         Set<DataType> realValidTypes = new LinkedHashSet<>();
-        Arrays.stream(validTypes).sorted(Comparator.comparing(DataType::name)).forEach(realValidTypes::add);
-        realValidTypes.add(DataTypes.NULL);
+        Arrays.stream(validTypes).sorted(Comparator.comparing(DataType::nameUpper)).forEach(realValidTypes::add);
+        realValidTypes.add(DataType.NULL);
         return realValidTypes;
     }
 
     public Set<DataType> sortedTypesSet(DataType[] validTypes, DataType... additionalTypes) {
         Set<DataType> mergedSet = new LinkedHashSet<>();
         Stream.concat(Stream.of(validTypes), Stream.of(additionalTypes))
-            .sorted(Comparator.comparing(DataType::name))
+            .sorted(Comparator.comparing(DataType::nameUpper))
             .forEach(mergedSet::add);
         return mergedSet;
     }
@@ -80,26 +79,26 @@ public Set<DataType> sortedTypesSet(DataType[] validTypes, DataType... additiona
      * All integer types (long, int, short, byte). For passing to {@link #required} or {@link #optional}.
      */
     protected static DataType[] integers() {
-        return DataTypes.types().stream().filter(DataType::isInteger).toArray(DataType[]::new);
+        return DataType.types().stream().filter(DataType::isInteger).toArray(DataType[]::new);
     }
 
     /**
      * All rational types (double, float, whatever). For passing to {@link #required} or {@link #optional}.
      */
     protected static DataType[] rationals() {
-        return DataTypes.types().stream().filter(DataType::isRational).toArray(DataType[]::new);
+        return DataType.types().stream().filter(DataType::isRational).toArray(DataType[]::new);
     }
 
     /**
      * All numeric types (integers and rationals.) For passing to {@link #required} or {@link #optional}.
      */
     protected static DataType[] numerics() {
-        return DataTypes.types().stream().filter(DataType::isNumeric).toArray(DataType[]::new);
+        return DataType.types().stream().filter(DataType::isNumeric).toArray(DataType[]::new);
     }
 
     protected final DataType[] representableNumerics() {
         // TODO numeric should only include representable numbers but that is a change for a followup
-        return DataTypes.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new);
+        return DataType.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new);
     }
 
     protected record ArgumentSpec(boolean optional, Set<DataType> validTypes) {}
@@ -107,7 +106,7 @@ protected record ArgumentSpec(boolean optional, Set<DataType> validTypes) {}
     public final void testResolveType() {
         List<ArgumentSpec> specs = argSpec();
         for (int mutArg = 0; mutArg < specs.size(); mutArg++) {
-            for (DataType mutArgType : DataTypes.types()) {
+            for (DataType mutArgType : DataType.types()) {
                 List<Expression> args = new ArrayList<>(specs.size());
                 for (int arg = 0; arg < specs.size(); arg++) {
                     if (mutArg == arg) {
@@ -161,11 +160,11 @@ protected Matcher<String> badTypeError(List<ArgumentSpec> spec, int badArgPositi
     }
 
     private String expectedTypeName(Set<DataType> validTypes) {
-        List<DataType> withoutNull = validTypes.stream().filter(t -> t != DataTypes.NULL).toList();
+        List<DataType> withoutNull = validTypes.stream().filter(t -> t != DataType.NULL).toList();
         if (withoutNull.equals(Arrays.asList(strings()))) {
             return "string";
         }
-        if (withoutNull.equals(Arrays.asList(integers())) || withoutNull.equals(List.of(DataTypes.INTEGER))) {
+        if (withoutNull.equals(Arrays.asList(integers())) || withoutNull.equals(List.of(DataType.INTEGER))) {
             return "integer";
         }
         if (withoutNull.equals(Arrays.asList(rationals()))) {
@@ -174,14 +173,14 @@ private String expectedTypeName(Set<DataType> validTypes) {
         if (withoutNull.equals(Arrays.asList(numerics())) || withoutNull.equals(Arrays.asList(representableNumerics()))) {
             return "numeric";
         }
-        if (withoutNull.equals(List.of(DataTypes.DATETIME))) {
+        if (withoutNull.equals(List.of(DataType.DATETIME))) {
             return "datetime";
         }
-        if (withoutNull.equals(List.of(DataTypes.IP))) {
+        if (withoutNull.equals(List.of(DataType.IP))) {
             return "ip";
         }
-        List<DataType> negations = Stream.concat(Stream.of(numerics()), Stream.of(DataTypes.DATE_PERIOD, DataTypes.TIME_DURATION))
-            .sorted(Comparator.comparing(DataType::name))
+        List<DataType> negations = Stream.concat(Stream.of(numerics()), Stream.of(DataType.DATE_PERIOD, DataType.TIME_DURATION))
+            .sorted(Comparator.comparing(DataType::nameUpper))
             .toList();
         if (withoutNull.equals(negations)) {
             return "numeric, date_period or time_duration";
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NamedExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NamedExpressionTests.java
index 536a24e6ce5c4..7843ad7eac8b7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NamedExpressionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NamedExpressionTests.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Location;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div;
@@ -53,7 +53,7 @@ public void testArithmeticFunctionName() {
     }
 
     public void testNameForArithmeticFunctionAppliedOnTableColumn() {
-        FieldAttribute fa = new FieldAttribute(EMPTY, "myField", new EsField("myESField", DataTypes.INTEGER, emptyMap(), true));
+        FieldAttribute fa = new FieldAttribute(EMPTY, "myField", new EsField("myESField", DataType.INTEGER, emptyMap(), true));
         String e = "myField  + 10";
         Add add = new Add(s(e), fa, l(10));
         assertEquals(e, add.sourceText());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/VaragsTestCaseBuilder.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/VaragsTestCaseBuilder.java
index 55a8c39756fcc..86cce2a66baf8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/VaragsTestCaseBuilder.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/VaragsTestCaseBuilder.java
@@ -10,7 +10,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
 
@@ -203,16 +202,16 @@ private void strings(List<TestCaseSupplier> suppliers) {
                 int paramCount = count;
                 suppliers.add(
                     new TestCaseSupplier(
-                        testCaseName(paramCount, multivalued, DataTypes.KEYWORD),
-                        dataTypes(paramCount, DataTypes.KEYWORD),
-                        () -> stringCase(DataTypes.KEYWORD, paramCount, multivalued)
+                        testCaseName(paramCount, multivalued, DataType.KEYWORD),
+                        dataTypes(paramCount, DataType.KEYWORD),
+                        () -> stringCase(DataType.KEYWORD, paramCount, multivalued)
                     )
                 );
                 suppliers.add(
                     new TestCaseSupplier(
-                        testCaseName(paramCount, multivalued, DataTypes.TEXT),
-                        dataTypes(paramCount, DataTypes.TEXT),
-                        () -> stringCase(DataTypes.TEXT, paramCount, multivalued)
+                        testCaseName(paramCount, multivalued, DataType.TEXT),
+                        dataTypes(paramCount, DataType.TEXT),
+                        () -> stringCase(DataType.TEXT, paramCount, multivalued)
                     )
                 );
             }
@@ -240,8 +239,8 @@ private void longs(List<TestCaseSupplier> suppliers) {
                 int paramCount = count;
                 suppliers.add(
                     new TestCaseSupplier(
-                        testCaseName(paramCount, multivalued, DataTypes.LONG),
-                        dataTypes(paramCount, DataTypes.LONG),
+                        testCaseName(paramCount, multivalued, DataType.LONG),
+                        dataTypes(paramCount, DataType.LONG),
                         () -> longCase(paramCount, multivalued)
                     )
                 );
@@ -257,14 +256,14 @@ private TestCaseSupplier.TestCase longCase(int paramCount, boolean multivalued)
                 List<Long> d = ESTestCase.randomList(1, 4, () -> ESTestCase.randomLong());
                 data[p] = d.stream().mapToLong(Long::longValue).toArray();
                 typedData.add(
-                    new TestCaseSupplier.TypedData(Arrays.stream(data[p]).mapToObj(Long::valueOf).toList(), DataTypes.LONG, "field" + p)
+                    new TestCaseSupplier.TypedData(Arrays.stream(data[p]).mapToObj(Long::valueOf).toList(), DataType.LONG, "field" + p)
                 );
             } else {
                 data[p] = new long[] { ESTestCase.randomLong() };
-                typedData.add(new TestCaseSupplier.TypedData(data[p][0], DataTypes.LONG, "field" + p));
+                typedData.add(new TestCaseSupplier.TypedData(data[p][0], DataType.LONG, "field" + p));
             }
         }
-        return testCase(typedData, expectedEvaluatorPrefix.apply("Long"), DataTypes.LONG, expectedLong.apply(data));
+        return testCase(typedData, expectedEvaluatorPrefix.apply("Long"), DataType.LONG, expectedLong.apply(data));
     }
 
     private void ints(List<TestCaseSupplier> suppliers) {
@@ -273,8 +272,8 @@ private void ints(List<TestCaseSupplier> suppliers) {
                 int paramCount = count;
                 suppliers.add(
                     new TestCaseSupplier(
-                        testCaseName(paramCount, multivalued, DataTypes.INTEGER),
-                        dataTypes(paramCount, DataTypes.INTEGER),
+                        testCaseName(paramCount, multivalued, DataType.INTEGER),
+                        dataTypes(paramCount, DataType.INTEGER),
                         () -> intCase(paramCount, multivalued)
                     )
                 );
@@ -289,13 +288,13 @@ private TestCaseSupplier.TestCase intCase(int paramCount, boolean multivalued) {
             if (multivalued) {
                 List<Integer> d = ESTestCase.randomList(1, 4, () -> ESTestCase.randomInt());
                 data[p] = d.stream().mapToInt(Integer::intValue).toArray();
-                typedData.add(new TestCaseSupplier.TypedData(d, DataTypes.INTEGER, "field" + p));
+                typedData.add(new TestCaseSupplier.TypedData(d, DataType.INTEGER, "field" + p));
             } else {
                 data[p] = new int[] { ESTestCase.randomInt() };
-                typedData.add(new TestCaseSupplier.TypedData(data[p][0], DataTypes.INTEGER, "field" + p));
+                typedData.add(new TestCaseSupplier.TypedData(data[p][0], DataType.INTEGER, "field" + p));
             }
         }
-        return testCase(typedData, expectedEvaluatorPrefix.apply("Int"), DataTypes.INTEGER, expectedInt.apply(data));
+        return testCase(typedData, expectedEvaluatorPrefix.apply("Int"), DataType.INTEGER, expectedInt.apply(data));
     }
 
     private void booleans(List<TestCaseSupplier> suppliers) {
@@ -304,8 +303,8 @@ private void booleans(List<TestCaseSupplier> suppliers) {
                 int paramCount = count;
                 suppliers.add(
                     new TestCaseSupplier(
-                        testCaseName(paramCount, multivalued, DataTypes.BOOLEAN),
-                        dataTypes(paramCount, DataTypes.BOOLEAN),
+                        testCaseName(paramCount, multivalued, DataType.BOOLEAN),
+                        dataTypes(paramCount, DataType.BOOLEAN),
                         () -> booleanCase(paramCount, multivalued)
                     )
                 );
@@ -325,13 +324,13 @@ private TestCaseSupplier.TestCase booleanCase(int paramCount, boolean multivalue
                     data[p][i] = ESTestCase.randomBoolean();
                     paramData.add(data[p][i]);
                 }
-                typedData.add(new TestCaseSupplier.TypedData(paramData, DataTypes.BOOLEAN, "field" + p));
+                typedData.add(new TestCaseSupplier.TypedData(paramData, DataType.BOOLEAN, "field" + p));
             } else {
                 data[p] = new boolean[] { ESTestCase.randomBoolean() };
-                typedData.add(new TestCaseSupplier.TypedData(data[p][0], DataTypes.BOOLEAN, "field" + p));
+                typedData.add(new TestCaseSupplier.TypedData(data[p][0], DataType.BOOLEAN, "field" + p));
             }
         }
-        return testCase(typedData, expectedEvaluatorPrefix.apply("Boolean"), DataTypes.BOOLEAN, expectedBoolean.apply(data));
+        return testCase(typedData, expectedEvaluatorPrefix.apply("Boolean"), DataType.BOOLEAN, expectedBoolean.apply(data));
     }
 
     private String testCaseName(int count, boolean multivalued, DataType type) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java
index 85d2ed7b30173..80a3cd48e5147 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 
 import java.util.List;
 
@@ -27,38 +27,38 @@ public void testElseValueExplicit() {
         assertThat(
             new Case(
                 Source.synthetic("case"),
-                field("first_cond", DataTypes.BOOLEAN),
-                List.of(field("v", DataTypes.LONG), field("e", DataTypes.LONG))
+                field("first_cond", DataType.BOOLEAN),
+                List.of(field("v", DataType.LONG), field("e", DataType.LONG))
             ).children(),
-            equalTo(List.of(field("first_cond", DataTypes.BOOLEAN), field("v", DataTypes.LONG), field("e", DataTypes.LONG)))
+            equalTo(List.of(field("first_cond", DataType.BOOLEAN), field("v", DataType.LONG), field("e", DataType.LONG)))
         );
     }
 
     public void testElseValueImplied() {
         assertThat(
-            new Case(Source.synthetic("case"), field("first_cond", DataTypes.BOOLEAN), List.of(field("v", DataTypes.LONG))).children(),
-            equalTo(List.of(field("first_cond", DataTypes.BOOLEAN), field("v", DataTypes.LONG)))
+            new Case(Source.synthetic("case"), field("first_cond", DataType.BOOLEAN), List.of(field("v", DataType.LONG))).children(),
+            equalTo(List.of(field("first_cond", DataType.BOOLEAN), field("v", DataType.LONG)))
         );
     }
 
     public void testPartialFoldDropsFirstFalse() {
         Case c = new Case(
             Source.synthetic("case"),
-            new Literal(Source.EMPTY, false, DataTypes.BOOLEAN),
-            List.of(field("first", DataTypes.LONG), field("last_cond", DataTypes.BOOLEAN), field("last", DataTypes.LONG))
+            new Literal(Source.EMPTY, false, DataType.BOOLEAN),
+            List.of(field("first", DataType.LONG), field("last_cond", DataType.BOOLEAN), field("last", DataType.LONG))
         );
         assertThat(c.foldable(), equalTo(false));
         assertThat(
             c.partiallyFold(),
-            equalTo(new Case(Source.synthetic("case"), field("last_cond", DataTypes.BOOLEAN), List.of(field("last", DataTypes.LONG))))
+            equalTo(new Case(Source.synthetic("case"), field("last_cond", DataType.BOOLEAN), List.of(field("last", DataType.LONG))))
         );
     }
 
     public void testPartialFoldNoop() {
         Case c = new Case(
             Source.synthetic("case"),
-            field("first_cond", DataTypes.BOOLEAN),
-            List.of(field("first", DataTypes.LONG), field("last", DataTypes.LONG))
+            field("first_cond", DataType.BOOLEAN),
+            List.of(field("first", DataType.LONG), field("last", DataType.LONG))
         );
         assertThat(c.foldable(), equalTo(false));
         assertThat(c.partiallyFold(), sameInstance(c));
@@ -67,22 +67,22 @@ public void testPartialFoldNoop() {
     public void testPartialFoldFirst() {
         Case c = new Case(
             Source.synthetic("case"),
-            new Literal(Source.EMPTY, true, DataTypes.BOOLEAN),
-            List.of(field("first", DataTypes.LONG), field("last", DataTypes.LONG))
+            new Literal(Source.EMPTY, true, DataType.BOOLEAN),
+            List.of(field("first", DataType.LONG), field("last", DataType.LONG))
         );
         assertThat(c.foldable(), equalTo(false));
-        assertThat(c.partiallyFold(), equalTo(field("first", DataTypes.LONG)));
+        assertThat(c.partiallyFold(), equalTo(field("first", DataType.LONG)));
     }
 
     public void testPartialFoldFirstAfterKeepingUnknown() {
         Case c = new Case(
             Source.synthetic("case"),
-            field("keep_me_cond", DataTypes.BOOLEAN),
+            field("keep_me_cond", DataType.BOOLEAN),
             List.of(
-                field("keep_me", DataTypes.LONG),
-                new Literal(Source.EMPTY, true, DataTypes.BOOLEAN),
-                field("first", DataTypes.LONG),
-                field("last", DataTypes.LONG)
+                field("keep_me", DataType.LONG),
+                new Literal(Source.EMPTY, true, DataType.BOOLEAN),
+                field("first", DataType.LONG),
+                field("last", DataType.LONG)
             )
         );
         assertThat(c.foldable(), equalTo(false));
@@ -91,8 +91,8 @@ public void testPartialFoldFirstAfterKeepingUnknown() {
             equalTo(
                 new Case(
                     Source.synthetic("case"),
-                    field("keep_me_cond", DataTypes.BOOLEAN),
-                    List.of(field("keep_me", DataTypes.LONG), field("first", DataTypes.LONG))
+                    field("keep_me_cond", DataType.BOOLEAN),
+                    List.of(field("keep_me", DataType.LONG), field("first", DataType.LONG))
                 )
             )
         );
@@ -101,57 +101,57 @@ public void testPartialFoldFirstAfterKeepingUnknown() {
     public void testPartialFoldSecond() {
         Case c = new Case(
             Source.synthetic("case"),
-            new Literal(Source.EMPTY, false, DataTypes.BOOLEAN),
+            new Literal(Source.EMPTY, false, DataType.BOOLEAN),
             List.of(
-                field("first", DataTypes.LONG),
-                new Literal(Source.EMPTY, true, DataTypes.BOOLEAN),
-                field("second", DataTypes.LONG),
-                field("last", DataTypes.LONG)
+                field("first", DataType.LONG),
+                new Literal(Source.EMPTY, true, DataType.BOOLEAN),
+                field("second", DataType.LONG),
+                field("last", DataType.LONG)
             )
         );
         assertThat(c.foldable(), equalTo(false));
-        assertThat(c.partiallyFold(), equalTo(field("second", DataTypes.LONG)));
+        assertThat(c.partiallyFold(), equalTo(field("second", DataType.LONG)));
     }
 
     public void testPartialFoldSecondAfterDroppingFalse() {
         Case c = new Case(
             Source.synthetic("case"),
-            new Literal(Source.EMPTY, false, DataTypes.BOOLEAN),
+            new Literal(Source.EMPTY, false, DataType.BOOLEAN),
             List.of(
-                field("first", DataTypes.LONG),
-                new Literal(Source.EMPTY, true, DataTypes.BOOLEAN),
-                field("second", DataTypes.LONG),
-                field("last", DataTypes.LONG)
+                field("first", DataType.LONG),
+                new Literal(Source.EMPTY, true, DataType.BOOLEAN),
+                field("second", DataType.LONG),
+                field("last", DataType.LONG)
             )
         );
         assertThat(c.foldable(), equalTo(false));
-        assertThat(c.partiallyFold(), equalTo(field("second", DataTypes.LONG)));
+        assertThat(c.partiallyFold(), equalTo(field("second", DataType.LONG)));
     }
 
     public void testPartialFoldLast() {
         Case c = new Case(
             Source.synthetic("case"),
-            new Literal(Source.EMPTY, false, DataTypes.BOOLEAN),
+            new Literal(Source.EMPTY, false, DataType.BOOLEAN),
             List.of(
-                field("first", DataTypes.LONG),
-                new Literal(Source.EMPTY, false, DataTypes.BOOLEAN),
-                field("second", DataTypes.LONG),
-                field("last", DataTypes.LONG)
+                field("first", DataType.LONG),
+                new Literal(Source.EMPTY, false, DataType.BOOLEAN),
+                field("second", DataType.LONG),
+                field("last", DataType.LONG)
             )
         );
         assertThat(c.foldable(), equalTo(false));
-        assertThat(c.partiallyFold(), equalTo(field("last", DataTypes.LONG)));
+        assertThat(c.partiallyFold(), equalTo(field("last", DataType.LONG)));
     }
 
     public void testPartialFoldLastAfterKeepingUnknown() {
         Case c = new Case(
             Source.synthetic("case"),
-            field("keep_me_cond", DataTypes.BOOLEAN),
+            field("keep_me_cond", DataType.BOOLEAN),
             List.of(
-                field("keep_me", DataTypes.LONG),
-                new Literal(Source.EMPTY, false, DataTypes.BOOLEAN),
-                field("first", DataTypes.LONG),
-                field("last", DataTypes.LONG)
+                field("keep_me", DataType.LONG),
+                new Literal(Source.EMPTY, false, DataType.BOOLEAN),
+                field("first", DataType.LONG),
+                field("last", DataType.LONG)
             )
         );
         assertThat(c.foldable(), equalTo(false));
@@ -160,8 +160,8 @@ public void testPartialFoldLastAfterKeepingUnknown() {
             equalTo(
                 new Case(
                     Source.synthetic("case"),
-                    field("keep_me_cond", DataTypes.BOOLEAN),
-                    List.of(field("keep_me", DataTypes.LONG), field("last", DataTypes.LONG))
+                    field("keep_me_cond", DataType.BOOLEAN),
+                    List.of(field("keep_me", DataType.LONG), field("last", DataType.LONG))
                 )
             )
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
index e14b69d627f2e..f24955eb4804a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -47,158 +46,158 @@ public CaseTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseS
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(
-            List.of(new TestCaseSupplier("keyword", List.of(DataTypes.BOOLEAN, DataTypes.KEYWORD, DataTypes.KEYWORD), () -> {
+            List.of(new TestCaseSupplier("keyword", List.of(DataType.BOOLEAN, DataType.KEYWORD, DataType.KEYWORD), () -> {
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(true, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(new BytesRef("a"), DataTypes.KEYWORD, "a"),
-                    new TestCaseSupplier.TypedData(new BytesRef("b"), DataTypes.KEYWORD, "b")
+                    new TestCaseSupplier.TypedData(true, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(new BytesRef("a"), DataType.KEYWORD, "a"),
+                    new TestCaseSupplier.TypedData(new BytesRef("b"), DataType.KEYWORD, "b")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=BYTES_REF, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=Attribute[channel=2]]",
-                    DataTypes.KEYWORD,
+                    DataType.KEYWORD,
                     equalTo(new BytesRef("a"))
                 );
-            }), new TestCaseSupplier("text", List.of(DataTypes.BOOLEAN, DataTypes.TEXT), () -> {
+            }), new TestCaseSupplier("text", List.of(DataType.BOOLEAN, DataType.TEXT), () -> {
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(new BytesRef("a"), DataTypes.TEXT, "trueValue")
+                    new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(new BytesRef("a"), DataType.TEXT, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=BYTES_REF, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.TEXT,
+                    DataType.TEXT,
                     nullValue()
                 );
-            }), new TestCaseSupplier("boolean", List.of(DataTypes.BOOLEAN, DataTypes.BOOLEAN), () -> {
+            }), new TestCaseSupplier("boolean", List.of(DataType.BOOLEAN, DataType.BOOLEAN), () -> {
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "trueValue")
+                    new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=BOOLEAN, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.BOOLEAN,
+                    DataType.BOOLEAN,
                     nullValue()
                 );
-            }), new TestCaseSupplier("date", List.of(DataTypes.BOOLEAN, DataTypes.DATETIME), () -> {
+            }), new TestCaseSupplier("date", List.of(DataType.BOOLEAN, DataType.DATETIME), () -> {
                 long value = randomNonNegativeLong();
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(true, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.DATETIME, "trueValue")
+                    new TestCaseSupplier.TypedData(true, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(value, DataType.DATETIME, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=LONG, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.DATETIME,
+                    DataType.DATETIME,
                     equalTo(value)
                 );
-            }), new TestCaseSupplier("double", List.of(DataTypes.BOOLEAN, DataTypes.DOUBLE), () -> {
+            }), new TestCaseSupplier("double", List.of(DataType.BOOLEAN, DataType.DOUBLE), () -> {
                 double value = randomDouble();
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(true, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.DOUBLE, "trueValue")
+                    new TestCaseSupplier.TypedData(true, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(value, DataType.DOUBLE, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=DOUBLE, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.DOUBLE,
+                    DataType.DOUBLE,
                     equalTo(value)
                 );
-            }), new TestCaseSupplier("integer", List.of(DataTypes.BOOLEAN, DataTypes.INTEGER), () -> {
+            }), new TestCaseSupplier("integer", List.of(DataType.BOOLEAN, DataType.INTEGER), () -> {
                 int value = randomInt();
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.INTEGER, "trueValue")
+                    new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(value, DataType.INTEGER, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=INT, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.INTEGER,
+                    DataType.INTEGER,
                     nullValue()
                 );
-            }), new TestCaseSupplier("long", List.of(DataTypes.BOOLEAN, DataTypes.LONG), () -> {
+            }), new TestCaseSupplier("long", List.of(DataType.BOOLEAN, DataType.LONG), () -> {
                 long value = randomLong();
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.LONG, "trueValue")
+                    new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(value, DataType.LONG, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=LONG, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.LONG,
+                    DataType.LONG,
                     nullValue()
                 );
-            }), new TestCaseSupplier("unsigned_long", List.of(DataTypes.BOOLEAN, DataTypes.UNSIGNED_LONG), () -> {
+            }), new TestCaseSupplier("unsigned_long", List.of(DataType.BOOLEAN, DataType.UNSIGNED_LONG), () -> {
                 BigInteger value = randomUnsignedLongBetween(BigInteger.ZERO, UNSIGNED_LONG_MAX);
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(true, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.UNSIGNED_LONG, "trueValue")
+                    new TestCaseSupplier.TypedData(true, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(value, DataType.UNSIGNED_LONG, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=LONG, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.UNSIGNED_LONG,
+                    DataType.UNSIGNED_LONG,
                     equalTo(value)
                 );
-            }), new TestCaseSupplier("ip", List.of(DataTypes.BOOLEAN, DataTypes.IP), () -> {
-                BytesRef value = (BytesRef) randomLiteral(DataTypes.IP).value();
+            }), new TestCaseSupplier("ip", List.of(DataType.BOOLEAN, DataType.IP), () -> {
+                BytesRef value = (BytesRef) randomLiteral(DataType.IP).value();
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(true, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.IP, "trueValue")
+                    new TestCaseSupplier.TypedData(true, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(value, DataType.IP, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=BYTES_REF, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.IP,
+                    DataType.IP,
                     equalTo(value)
                 );
-            }), new TestCaseSupplier("version", List.of(DataTypes.BOOLEAN, DataTypes.VERSION), () -> {
-                BytesRef value = (BytesRef) randomLiteral(DataTypes.VERSION).value();
+            }), new TestCaseSupplier("version", List.of(DataType.BOOLEAN, DataType.VERSION), () -> {
+                BytesRef value = (BytesRef) randomLiteral(DataType.VERSION).value();
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.VERSION, "trueValue")
+                    new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(value, DataType.VERSION, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=BYTES_REF, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.VERSION,
+                    DataType.VERSION,
                     nullValue()
                 );
-            }), new TestCaseSupplier("cartesian_point", List.of(DataTypes.BOOLEAN, DataTypes.CARTESIAN_POINT), () -> {
-                BytesRef value = (BytesRef) randomLiteral(DataTypes.CARTESIAN_POINT).value();
+            }), new TestCaseSupplier("cartesian_point", List.of(DataType.BOOLEAN, DataType.CARTESIAN_POINT), () -> {
+                BytesRef value = (BytesRef) randomLiteral(DataType.CARTESIAN_POINT).value();
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.CARTESIAN_POINT, "trueValue")
+                    new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(value, DataType.CARTESIAN_POINT, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=BYTES_REF, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.CARTESIAN_POINT,
+                    DataType.CARTESIAN_POINT,
                     nullValue()
                 );
-            }), new TestCaseSupplier("geo_point", List.of(DataTypes.BOOLEAN, DataTypes.GEO_POINT), () -> {
-                BytesRef value = (BytesRef) randomLiteral(DataTypes.GEO_POINT).value();
+            }), new TestCaseSupplier("geo_point", List.of(DataType.BOOLEAN, DataType.GEO_POINT), () -> {
+                BytesRef value = (BytesRef) randomLiteral(DataType.GEO_POINT).value();
                 List<TestCaseSupplier.TypedData> typedData = List.of(
-                    new TestCaseSupplier.TypedData(true, DataTypes.BOOLEAN, "cond"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.GEO_POINT, "trueValue")
+                    new TestCaseSupplier.TypedData(true, DataType.BOOLEAN, "cond"),
+                    new TestCaseSupplier.TypedData(value, DataType.GEO_POINT, "trueValue")
                 );
                 return new TestCaseSupplier.TestCase(
                     typedData,
                     "CaseEvaluator[resultType=BYTES_REF, conditions=[ConditionEvaluator[condition=Attribute[channel=0], "
                         + "value=Attribute[channel=1]]], elseVal=LiteralsEvaluator[lit=null]]",
-                    DataTypes.GEO_POINT,
+                    DataType.GEO_POINT,
                     equalTo(value)
                 );
             }))
@@ -271,15 +270,15 @@ public void testCase(Function<Case, Object> toValue) {
         assertEquals(3, toValue.apply(caseExpr(false, 1, false, 2, 3)));
         assertNull(toValue.apply(caseExpr(true, null, 1)));
         assertEquals(1, toValue.apply(caseExpr(false, null, 1)));
-        assertEquals(1, toValue.apply(caseExpr(false, field("ignored", DataTypes.INTEGER), 1)));
-        assertEquals(1, toValue.apply(caseExpr(true, 1, field("ignored", DataTypes.INTEGER))));
+        assertEquals(1, toValue.apply(caseExpr(false, field("ignored", DataType.INTEGER), 1)));
+        assertEquals(1, toValue.apply(caseExpr(true, 1, field("ignored", DataType.INTEGER))));
     }
 
     public void testIgnoreLeadingNulls() {
-        assertEquals(DataTypes.INTEGER, resolveType(false, null, 1));
-        assertEquals(DataTypes.INTEGER, resolveType(false, null, false, null, false, 2, null));
-        assertEquals(DataTypes.NULL, resolveType(false, null, null));
-        assertEquals(DataTypes.BOOLEAN, resolveType(false, null, field("bool", DataTypes.BOOLEAN)));
+        assertEquals(DataType.INTEGER, resolveType(false, null, 1));
+        assertEquals(DataType.INTEGER, resolveType(false, null, false, null, false, 2, null));
+        assertEquals(DataType.NULL, resolveType(false, null, null));
+        assertEquals(DataType.BOOLEAN, resolveType(false, null, field("bool", DataType.BOOLEAN)));
     }
 
     public void testCaseWithInvalidCondition() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java
index 38092791f6d6a..9376849d8136c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.VaragsTestCaseBuilder;
@@ -43,14 +43,14 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(
             new TestCaseSupplier(
                 "(a, b)",
-                List.of(DataTypes.KEYWORD, DataTypes.KEYWORD),
+                List.of(DataType.KEYWORD, DataType.KEYWORD),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(new BytesRef("a"), DataTypes.KEYWORD, "a"),
-                        new TestCaseSupplier.TypedData(new BytesRef("b"), DataTypes.KEYWORD, "b")
+                        new TestCaseSupplier.TypedData(new BytesRef("a"), DataType.KEYWORD, "a"),
+                        new TestCaseSupplier.TypedData(new BytesRef("b"), DataType.KEYWORD, "b")
                     ),
                     "GreatestBytesRefEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]",
-                    DataTypes.KEYWORD,
+                    DataType.KEYWORD,
                     equalTo(new BytesRef("b"))
                 )
             )
@@ -58,14 +58,14 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(
             new TestCaseSupplier(
                 "(a, b)",
-                List.of(DataTypes.VERSION, DataTypes.VERSION),
+                List.of(DataType.VERSION, DataType.VERSION),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(new BytesRef("1"), DataTypes.VERSION, "a"),
-                        new TestCaseSupplier.TypedData(new BytesRef("2"), DataTypes.VERSION, "b")
+                        new TestCaseSupplier.TypedData(new BytesRef("1"), DataType.VERSION, "a"),
+                        new TestCaseSupplier.TypedData(new BytesRef("2"), DataType.VERSION, "b")
                     ),
                     "GreatestBytesRefEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]",
-                    DataTypes.VERSION,
+                    DataType.VERSION,
                     equalTo(new BytesRef("2"))
                 )
             )
@@ -73,14 +73,14 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(
             new TestCaseSupplier(
                 "(a, b)",
-                List.of(DataTypes.IP, DataTypes.IP),
+                List.of(DataType.IP, DataType.IP),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(new BytesRef("127.0.0.1"), DataTypes.IP, "a"),
-                        new TestCaseSupplier.TypedData(new BytesRef("127.0.0.2"), DataTypes.IP, "b")
+                        new TestCaseSupplier.TypedData(new BytesRef("127.0.0.1"), DataType.IP, "a"),
+                        new TestCaseSupplier.TypedData(new BytesRef("127.0.0.2"), DataType.IP, "b")
                     ),
                     "GreatestBytesRefEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]",
-                    DataTypes.IP,
+                    DataType.IP,
                     equalTo(new BytesRef("127.0.0.2"))
                 )
             )
@@ -88,14 +88,14 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(
             new TestCaseSupplier(
                 "(a, b)",
-                List.of(DataTypes.DOUBLE, DataTypes.DOUBLE),
+                List.of(DataType.DOUBLE, DataType.DOUBLE),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(1d, DataTypes.DOUBLE, "a"),
-                        new TestCaseSupplier.TypedData(2d, DataTypes.DOUBLE, "b")
+                        new TestCaseSupplier.TypedData(1d, DataType.DOUBLE, "a"),
+                        new TestCaseSupplier.TypedData(2d, DataType.DOUBLE, "b")
                     ),
                     "GreatestDoubleEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]",
-                    DataTypes.DOUBLE,
+                    DataType.DOUBLE,
                     equalTo(2d)
                 )
             )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java
index cadfb1e6d1c6f..0881b871c30f6 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.VaragsTestCaseBuilder;
@@ -42,14 +42,14 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(
             new TestCaseSupplier(
                 "(a, b)",
-                List.of(DataTypes.KEYWORD, DataTypes.KEYWORD),
+                List.of(DataType.KEYWORD, DataType.KEYWORD),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(new BytesRef("a"), DataTypes.KEYWORD, "a"),
-                        new TestCaseSupplier.TypedData(new BytesRef("b"), DataTypes.KEYWORD, "b")
+                        new TestCaseSupplier.TypedData(new BytesRef("a"), DataType.KEYWORD, "a"),
+                        new TestCaseSupplier.TypedData(new BytesRef("b"), DataType.KEYWORD, "b")
                     ),
                     "LeastBytesRefEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]",
-                    DataTypes.KEYWORD,
+                    DataType.KEYWORD,
                     equalTo(new BytesRef("a"))
                 )
             )
@@ -57,14 +57,14 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(
             new TestCaseSupplier(
                 "(a, b)",
-                List.of(DataTypes.VERSION, DataTypes.VERSION),
+                List.of(DataType.VERSION, DataType.VERSION),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(new BytesRef("1"), DataTypes.VERSION, "a"),
-                        new TestCaseSupplier.TypedData(new BytesRef("2"), DataTypes.VERSION, "b")
+                        new TestCaseSupplier.TypedData(new BytesRef("1"), DataType.VERSION, "a"),
+                        new TestCaseSupplier.TypedData(new BytesRef("2"), DataType.VERSION, "b")
                     ),
                     "LeastBytesRefEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]",
-                    DataTypes.VERSION,
+                    DataType.VERSION,
                     equalTo(new BytesRef("1"))
                 )
             )
@@ -72,14 +72,14 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(
             new TestCaseSupplier(
                 "(a, b)",
-                List.of(DataTypes.IP, DataTypes.IP),
+                List.of(DataType.IP, DataType.IP),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(new BytesRef("127.0.0.1"), DataTypes.IP, "a"),
-                        new TestCaseSupplier.TypedData(new BytesRef("127.0.0.2"), DataTypes.IP, "b")
+                        new TestCaseSupplier.TypedData(new BytesRef("127.0.0.1"), DataType.IP, "a"),
+                        new TestCaseSupplier.TypedData(new BytesRef("127.0.0.2"), DataType.IP, "b")
                     ),
                     "LeastBytesRefEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]",
-                    DataTypes.IP,
+                    DataType.IP,
                     equalTo(new BytesRef("127.0.0.1"))
                 )
             )
@@ -87,14 +87,14 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(
             new TestCaseSupplier(
                 "(a, b)",
-                List.of(DataTypes.DOUBLE, DataTypes.DOUBLE),
+                List.of(DataType.DOUBLE, DataType.DOUBLE),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(1d, DataTypes.DOUBLE, "a"),
-                        new TestCaseSupplier.TypedData(2d, DataTypes.DOUBLE, "b")
+                        new TestCaseSupplier.TypedData(1d, DataType.DOUBLE, "a"),
+                        new TestCaseSupplier.TypedData(2d, DataType.DOUBLE, "b")
                     ),
                     "LeastDoubleEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]",
-                    DataTypes.DOUBLE,
+                    DataType.DOUBLE,
                     equalTo(1d)
                 )
             )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java
index 62b0070e8a336..2096d9cec75b1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -35,22 +35,22 @@ public FromBase64Tests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> tes
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.KEYWORD), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD), () -> {
             BytesRef input = new BytesRef(randomAlphaOfLength(6));
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(input, DataTypes.KEYWORD, "string")),
+                List.of(new TestCaseSupplier.TypedData(input, DataType.KEYWORD, "string")),
                 "FromBase64Evaluator[field=Attribute[channel=0]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(Base64.getDecoder().decode(input.utf8ToString().getBytes(StandardCharsets.UTF_8))))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.TEXT), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT), () -> {
             BytesRef input = new BytesRef(randomAlphaOfLength(54));
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(input, DataTypes.TEXT, "string")),
+                List.of(new TestCaseSupplier.TypedData(input, DataType.TEXT, "string")),
                 "FromBase64Evaluator[field=Attribute[channel=0]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(Base64.getDecoder().decode(input.utf8ToString().getBytes(StandardCharsets.UTF_8))))
             );
         }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java
index fe25e65befab5..dd35e04708c9f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -35,22 +35,22 @@ public ToBase64Tests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testC
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.KEYWORD), () -> {
-            BytesRef input = (BytesRef) randomLiteral(DataTypes.KEYWORD).value();
+        suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD), () -> {
+            BytesRef input = (BytesRef) randomLiteral(DataType.KEYWORD).value();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(input, DataTypes.KEYWORD, "string")),
+                List.of(new TestCaseSupplier.TypedData(input, DataType.KEYWORD, "string")),
                 "ToBase64Evaluator[field=Attribute[channel=0]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(Base64.getEncoder().encode(input.utf8ToString().getBytes(StandardCharsets.UTF_8))))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.TEXT), () -> {
-            BytesRef input = (BytesRef) randomLiteral(DataTypes.TEXT).value();
+        suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT), () -> {
+            BytesRef input = (BytesRef) randomLiteral(DataType.TEXT).value();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(input, DataTypes.TEXT, "string")),
+                List.of(new TestCaseSupplier.TypedData(input, DataType.TEXT, "string")),
                 "ToBase64Evaluator[field=Attribute[channel=0]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(Base64.getEncoder().encode(input.utf8ToString().getBytes(StandardCharsets.UTF_8))))
             );
         }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java
index 8c37f7055066f..3a25ad6b56d0c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -34,12 +34,12 @@ public static Iterable<Object[]> parameters() {
         final String read = "Attribute[channel=0]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryBoolean(suppliers, read, DataTypes.BOOLEAN, b -> b, emptyList());
+        TestCaseSupplier.forUnaryBoolean(suppliers, read, DataType.BOOLEAN, b -> b, emptyList());
 
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "ToBooleanFromIntEvaluator[field=" + read + "]",
-            DataTypes.BOOLEAN,
+            DataType.BOOLEAN,
             i -> i != 0,
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
@@ -48,7 +48,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             "ToBooleanFromLongEvaluator[field=" + read + "]",
-            DataTypes.BOOLEAN,
+            DataType.BOOLEAN,
             l -> l != 0,
             Long.MIN_VALUE,
             Long.MAX_VALUE,
@@ -57,7 +57,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "ToBooleanFromUnsignedLongEvaluator[field=" + read + "]",
-            DataTypes.BOOLEAN,
+            DataType.BOOLEAN,
             ul -> ul.compareTo(BigInteger.ZERO) != 0,
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
@@ -66,7 +66,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "ToBooleanFromDoubleEvaluator[field=" + read + "]",
-            DataTypes.BOOLEAN,
+            DataType.BOOLEAN,
             d -> d != 0d,
             Double.NEGATIVE_INFINITY,
             Double.POSITIVE_INFINITY,
@@ -75,7 +75,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryStrings(
             suppliers,
             "ToBooleanFromStringEvaluator[field=" + read + "]",
-            DataTypes.BOOLEAN,
+            DataType.BOOLEAN,
             bytesRef -> String.valueOf(bytesRef).toLowerCase(Locale.ROOT).equals("true"),
             emptyList()
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
index 6e63a14a0ad7c..b520e559c45d7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -40,12 +39,12 @@ public static Iterable<Object[]> parameters() {
         final Function<String, String> evaluatorName = s -> "ToCartesianPoint" + s + "Evaluator[field=" + attribute + "]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryCartesianPoint(suppliers, attribute, DataTypes.CARTESIAN_POINT, v -> v, List.of());
+        TestCaseSupplier.forUnaryCartesianPoint(suppliers, attribute, DataType.CARTESIAN_POINT, v -> v, List.of());
         // random strings that don't look like a cartesian point
         TestCaseSupplier.forUnaryStrings(
             suppliers,
             evaluatorName.apply("FromString"),
-            DataTypes.CARTESIAN_POINT,
+            DataType.CARTESIAN_POINT,
             bytesRef -> null,
             bytesRef -> {
                 var exception = expectThrows(Exception.class, () -> CARTESIAN.wktToWkb(bytesRef.utf8ToString()));
@@ -56,7 +55,7 @@ public static Iterable<Object[]> parameters() {
             }
         );
         // strings that are cartesian point representations
-        for (DataType dt : List.of(DataTypes.KEYWORD, DataTypes.TEXT)) {
+        for (DataType dt : List.of(DataType.KEYWORD, DataType.TEXT)) {
             TestCaseSupplier.unary(
                 suppliers,
                 evaluatorName.apply("FromString"),
@@ -67,7 +66,7 @@ public static Iterable<Object[]> parameters() {
                         dt
                     )
                 ),
-                DataTypes.CARTESIAN_POINT,
+                DataType.CARTESIAN_POINT,
                 bytesRef -> CARTESIAN.wktToWkb(((BytesRef) bytesRef).utf8ToString()),
                 List.of()
             );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
index 9ffb8bd82a0a8..9eb1155a209a1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -40,13 +39,13 @@ public static Iterable<Object[]> parameters() {
         final Function<String, String> evaluatorName = s -> "ToCartesianShape" + s + "Evaluator[field=" + attribute + "]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryCartesianPoint(suppliers, attribute, DataTypes.CARTESIAN_SHAPE, v -> v, List.of());
-        TestCaseSupplier.forUnaryCartesianShape(suppliers, attribute, DataTypes.CARTESIAN_SHAPE, v -> v, List.of());
+        TestCaseSupplier.forUnaryCartesianPoint(suppliers, attribute, DataType.CARTESIAN_SHAPE, v -> v, List.of());
+        TestCaseSupplier.forUnaryCartesianShape(suppliers, attribute, DataType.CARTESIAN_SHAPE, v -> v, List.of());
         // random strings that don't look like a cartesian shape
         TestCaseSupplier.forUnaryStrings(
             suppliers,
             evaluatorName.apply("FromString"),
-            DataTypes.CARTESIAN_SHAPE,
+            DataType.CARTESIAN_SHAPE,
             bytesRef -> null,
             bytesRef -> {
                 var exception = expectThrows(Exception.class, () -> CARTESIAN.wktToWkb(bytesRef.utf8ToString()));
@@ -57,7 +56,7 @@ public static Iterable<Object[]> parameters() {
             }
         );
         // strings that are cartesian_shape representations
-        for (DataType dt : List.of(DataTypes.KEYWORD, DataTypes.TEXT)) {
+        for (DataType dt : List.of(DataType.KEYWORD, DataType.TEXT)) {
             TestCaseSupplier.unary(
                 suppliers,
                 evaluatorName.apply("FromString"),
@@ -68,7 +67,7 @@ public static Iterable<Object[]> parameters() {
                         dt
                     )
                 ),
-                DataTypes.CARTESIAN_SHAPE,
+                DataType.CARTESIAN_SHAPE,
                 bytesRef -> CARTESIAN.wktToWkb(((BytesRef) bytesRef).utf8ToString()),
                 List.of()
             );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
index 4660a9a1c86a2..0ef931710422e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -36,22 +36,22 @@ public static Iterable<Object[]> parameters() {
         final String read = "Attribute[channel=0]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryDatetime(suppliers, read, DataTypes.DATETIME, Instant::toEpochMilli, emptyList());
+        TestCaseSupplier.forUnaryDatetime(suppliers, read, DataType.DATETIME, Instant::toEpochMilli, emptyList());
 
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "ToLongFromIntEvaluator[field=" + read + "]",
-            DataTypes.DATETIME,
+            DataType.DATETIME,
             i -> ((Integer) i).longValue(),
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
             emptyList()
         );
-        TestCaseSupplier.forUnaryLong(suppliers, read, DataTypes.DATETIME, l -> l, Long.MIN_VALUE, Long.MAX_VALUE, emptyList());
+        TestCaseSupplier.forUnaryLong(suppliers, read, DataType.DATETIME, l -> l, Long.MIN_VALUE, Long.MAX_VALUE, emptyList());
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "ToLongFromUnsignedLongEvaluator[field=" + read + "]",
-            DataTypes.DATETIME,
+            DataType.DATETIME,
             BigInteger::longValueExact,
             BigInteger.ZERO,
             BigInteger.valueOf(Long.MAX_VALUE),
@@ -60,7 +60,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "ToLongFromUnsignedLongEvaluator[field=" + read + "]",
-            DataTypes.DATETIME,
+            DataType.DATETIME,
             bi -> null,
             BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TWO),
             UNSIGNED_LONG_MAX,
@@ -72,7 +72,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "ToLongFromDoubleEvaluator[field=" + read + "]",
-            DataTypes.DATETIME,
+            DataType.DATETIME,
             d -> null,
             Double.NEGATIVE_INFINITY,
             -9.223372036854777E18, // a "convenient" value smaller than `(double) Long.MIN_VALUE` (== ...776E18)
@@ -84,7 +84,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "ToLongFromDoubleEvaluator[field=" + read + "]",
-            DataTypes.DATETIME,
+            DataType.DATETIME,
             d -> null,
             9.223372036854777E18, // a "convenient" value larger than `(double) Long.MAX_VALUE` (== ...776E18)
             Double.POSITIVE_INFINITY,
@@ -96,7 +96,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryStrings(
             suppliers,
             "ToDatetimeFromStringEvaluator[field=" + read + "]",
-            DataTypes.DATETIME,
+            DataType.DATETIME,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -114,10 +114,10 @@ public static Iterable<Object[]> parameters() {
                     "<date string>",
                     // millis past "0001-01-01T00:00:00.000Z" to match the default formatter
                     () -> new BytesRef(randomDateString(-62135596800000L, 253402300799999L)),
-                    DataTypes.KEYWORD
+                    DataType.KEYWORD
                 )
             ),
-            DataTypes.DATETIME,
+            DataType.DATETIME,
             bytesRef -> DEFAULT_DATE_TIME_FORMATTER.parseMillis(((BytesRef) bytesRef).utf8ToString()),
             emptyList()
         );
@@ -129,10 +129,10 @@ public static Iterable<Object[]> parameters() {
                     "<date string before 0001-01-01T00:00:00.000Z>",
                     // millis before "0001-01-01T00:00:00.000Z"
                     () -> new BytesRef(randomDateString(Long.MIN_VALUE, -62135596800001L)),
-                    DataTypes.KEYWORD
+                    DataType.KEYWORD
                 )
             ),
-            DataTypes.DATETIME,
+            DataType.DATETIME,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -149,10 +149,10 @@ public static Iterable<Object[]> parameters() {
                     "<date string before 0001-01-01T00:00:00.000Z>",
                     // millis before "0001-01-01T00:00:00.000Z"
                     () -> new BytesRef(randomDateString(253402300800000L, Long.MAX_VALUE)),
-                    DataTypes.KEYWORD
+                    DataType.KEYWORD
                 )
             ),
-            DataTypes.DATETIME,
+            DataType.DATETIME,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java
index 2368d52659898..b7cb03879fd6f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -36,7 +36,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             evaluatorName.apply("ToDoubleFromIntEvaluator"),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::toDegrees,
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
@@ -45,7 +45,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             evaluatorName.apply("ToDoubleFromLongEvaluator"),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::toDegrees,
             Long.MIN_VALUE,
             Long.MAX_VALUE,
@@ -54,13 +54,13 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             evaluatorName.apply("ToDoubleFromUnsignedLongEvaluator"),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             ul -> Math.toDegrees(ul.doubleValue()),
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
             List.of()
         );
-        TestCaseSupplier.forUnaryDouble(suppliers, "ToDegreesEvaluator[field=Attribute[channel=0]]", DataTypes.DOUBLE, d -> {
+        TestCaseSupplier.forUnaryDouble(suppliers, "ToDegreesEvaluator[field=Attribute[channel=0]]", DataType.DOUBLE, d -> {
             double deg = Math.toDegrees(d);
             return Double.isNaN(deg) || Double.isInfinite(deg) ? null : deg;
         }, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, d -> {
@@ -76,12 +76,12 @@ public static Iterable<Object[]> parameters() {
             suppliers,
             "ToDegreesEvaluator[field=Attribute[channel=0]]",
             List.of(
-                new TestCaseSupplier.TypedDataSupplier("Double.MAX_VALUE", () -> Double.MAX_VALUE, DataTypes.DOUBLE),
-                new TestCaseSupplier.TypedDataSupplier("-Double.MAX_VALUE", () -> -Double.MAX_VALUE, DataTypes.DOUBLE),
-                new TestCaseSupplier.TypedDataSupplier("Double.POSITIVE_INFINITY", () -> Double.POSITIVE_INFINITY, DataTypes.DOUBLE),
-                new TestCaseSupplier.TypedDataSupplier("Double.NEGATIVE_INFINITY", () -> Double.NEGATIVE_INFINITY, DataTypes.DOUBLE)
+                new TestCaseSupplier.TypedDataSupplier("Double.MAX_VALUE", () -> Double.MAX_VALUE, DataType.DOUBLE),
+                new TestCaseSupplier.TypedDataSupplier("-Double.MAX_VALUE", () -> -Double.MAX_VALUE, DataType.DOUBLE),
+                new TestCaseSupplier.TypedDataSupplier("Double.POSITIVE_INFINITY", () -> Double.POSITIVE_INFINITY, DataType.DOUBLE),
+                new TestCaseSupplier.TypedDataSupplier("Double.NEGATIVE_INFINITY", () -> Double.NEGATIVE_INFINITY, DataType.DOUBLE)
             ),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             d -> null,
             d -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
index 24f7952c67f6a..6438a8422a664 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
@@ -41,23 +41,23 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             read,
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             d -> d,
             Double.NEGATIVE_INFINITY,
             Double.POSITIVE_INFINITY,
             List.of()
         );
 
-        TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataTypes.DOUBLE, b -> b ? 1d : 0d, List.of());
+        TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataType.DOUBLE, b -> b ? 1d : 0d, List.of());
         TestCaseSupplier.forUnaryDatetime(
             suppliers,
             evaluatorName.apply("Long"),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             i -> (double) i.toEpochMilli(),
             List.of()
         );
         // random strings that don't look like a double
-        TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("String"), DataTypes.DOUBLE, bytesRef -> null, bytesRef -> {
+        TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("String"), DataType.DOUBLE, bytesRef -> null, bytesRef -> {
             var exception = expectThrows(
                 InvalidArgumentException.class,
                 () -> EsqlDataTypeConverter.stringToDouble(bytesRef.utf8ToString())
@@ -70,7 +70,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             evaluatorName.apply("UnsignedLong"),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             BigInteger::doubleValue,
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
@@ -79,7 +79,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             evaluatorName.apply("Long"),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             l -> (double) l,
             Long.MIN_VALUE,
             Long.MAX_VALUE,
@@ -88,7 +88,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             evaluatorName.apply("Int"),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             i -> (double) i,
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
@@ -105,11 +105,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             bytesRef -> Double.valueOf(((BytesRef) bytesRef).utf8ToString()),
             List.of()
         );
@@ -117,24 +117,24 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.unary(
             suppliers,
             "Attribute[channel=0]",
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomDouble, DataTypes.COUNTER_DOUBLE)),
-            DataTypes.DOUBLE,
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomDouble, DataType.COUNTER_DOUBLE)),
+            DataType.DOUBLE,
             l -> l,
             List.of()
         );
         TestCaseSupplier.unary(
             suppliers,
             evaluatorName.apply("Integer"),
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", () -> randomInt(1000), DataTypes.COUNTER_INTEGER)),
-            DataTypes.DOUBLE,
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", () -> randomInt(1000), DataType.COUNTER_INTEGER)),
+            DataType.DOUBLE,
             l -> ((Integer) l).doubleValue(),
             List.of()
         );
         TestCaseSupplier.unary(
             suppliers,
             evaluatorName.apply("Long"),
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", () -> randomLongBetween(1, 1000), DataTypes.COUNTER_LONG)),
-            DataTypes.DOUBLE,
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", () -> randomLongBetween(1, 1000), DataType.COUNTER_LONG)),
+            DataType.DOUBLE,
             l -> ((Long) l).doubleValue(),
             List.of()
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
index 4b7d78980a440..e1af4441b3c5f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -40,9 +39,9 @@ public static Iterable<Object[]> parameters() {
         final Function<String, String> evaluatorName = s -> "ToGeoPoint" + s + "Evaluator[field=" + attribute + "]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryGeoPoint(suppliers, attribute, DataTypes.GEO_POINT, v -> v, List.of());
+        TestCaseSupplier.forUnaryGeoPoint(suppliers, attribute, DataType.GEO_POINT, v -> v, List.of());
         // random strings that don't look like a geo point
-        TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("FromString"), DataTypes.GEO_POINT, bytesRef -> null, bytesRef -> {
+        TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("FromString"), DataType.GEO_POINT, bytesRef -> null, bytesRef -> {
             var exception = expectThrows(Exception.class, () -> GEO.wktToWkb(bytesRef.utf8ToString()));
             return List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -50,7 +49,7 @@ public static Iterable<Object[]> parameters() {
             );
         });
         // strings that are geo point representations
-        for (DataType dt : List.of(DataTypes.KEYWORD, DataTypes.TEXT)) {
+        for (DataType dt : List.of(DataType.KEYWORD, DataType.TEXT)) {
             TestCaseSupplier.unary(
                 suppliers,
                 evaluatorName.apply("FromString"),
@@ -61,7 +60,7 @@ public static Iterable<Object[]> parameters() {
                         dt
                     )
                 ),
-                DataTypes.GEO_POINT,
+                DataType.GEO_POINT,
                 bytesRef -> GEO.wktToWkb(((BytesRef) bytesRef).utf8ToString()),
                 List.of()
             );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
index 0b1f01ee24183..291708e94888c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -40,10 +39,10 @@ public static Iterable<Object[]> parameters() {
         final Function<String, String> evaluatorName = s -> "ToGeoShape" + s + "Evaluator[field=" + attribute + "]";
         final List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryGeoPoint(suppliers, attribute, DataTypes.GEO_SHAPE, v -> v, List.of());
-        TestCaseSupplier.forUnaryGeoShape(suppliers, attribute, DataTypes.GEO_SHAPE, v -> v, List.of());
+        TestCaseSupplier.forUnaryGeoPoint(suppliers, attribute, DataType.GEO_SHAPE, v -> v, List.of());
+        TestCaseSupplier.forUnaryGeoShape(suppliers, attribute, DataType.GEO_SHAPE, v -> v, List.of());
         // random strings that don't look like a geo shape
-        TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("FromString"), DataTypes.GEO_SHAPE, bytesRef -> null, bytesRef -> {
+        TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("FromString"), DataType.GEO_SHAPE, bytesRef -> null, bytesRef -> {
             var exception = expectThrows(Exception.class, () -> GEO.wktToWkb(bytesRef.utf8ToString()));
             return List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -51,7 +50,7 @@ public static Iterable<Object[]> parameters() {
             );
         });
         // strings that are geo_shape representations
-        for (DataType dt : List.of(DataTypes.KEYWORD, DataTypes.TEXT)) {
+        for (DataType dt : List.of(DataType.KEYWORD, DataType.TEXT)) {
             TestCaseSupplier.unary(
                 suppliers,
                 evaluatorName.apply("FromString"),
@@ -62,7 +61,7 @@ public static Iterable<Object[]> parameters() {
                         dt
                     )
                 ),
-                DataTypes.GEO_SHAPE,
+                DataType.GEO_SHAPE,
                 bytesRef -> GEO.wktToWkb(((BytesRef) bytesRef).utf8ToString()),
                 List.of()
             );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
index 8f106db3e97a0..415d9ea0a4a70 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -38,13 +38,13 @@ public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
 
         // convert from IP to IP
-        TestCaseSupplier.forUnaryIp(suppliers, read, DataTypes.IP, v -> v, List.of());
+        TestCaseSupplier.forUnaryIp(suppliers, read, DataType.IP, v -> v, List.of());
 
         // convert random string (i.e. not an IP representation) to IP `null`, with warnings.
         TestCaseSupplier.forUnaryStrings(
             suppliers,
             stringEvaluator,
-            DataTypes.IP,
+            DataType.IP,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -57,7 +57,7 @@ public static Iterable<Object[]> parameters() {
             suppliers,
             stringEvaluator,
             validIPsAsStrings(),
-            DataTypes.IP,
+            DataType.IP,
             bytesRef -> parseIP(((BytesRef) bytesRef).utf8ToString()),
             emptyList()
         );
@@ -73,16 +73,16 @@ protected Expression build(Source source, List<Expression> args) {
 
     private static List<TestCaseSupplier.TypedDataSupplier> validIPsAsStrings() {
         return List.of(
-            new TestCaseSupplier.TypedDataSupplier("<127.0.0.1 ip>", () -> new BytesRef("127.0.0.1"), DataTypes.KEYWORD),
+            new TestCaseSupplier.TypedDataSupplier("<127.0.0.1 ip>", () -> new BytesRef("127.0.0.1"), DataType.KEYWORD),
             new TestCaseSupplier.TypedDataSupplier(
                 "<ipv4>",
                 () -> new BytesRef(NetworkAddress.format(ESTestCase.randomIp(true))),
-                DataTypes.KEYWORD
+                DataType.KEYWORD
             ),
             new TestCaseSupplier.TypedDataSupplier(
                 "<ipv6>",
                 () -> new BytesRef(NetworkAddress.format(ESTestCase.randomIp(false))),
-                DataTypes.TEXT
+                DataType.TEXT
             )
         );
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
index ffac75f1ac338..83bdaf2f2d304 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -38,16 +38,16 @@ public static Iterable<Object[]> parameters() {
         Function<String, String> evaluatorName = s -> "ToIntegerFrom" + s + "Evaluator[field=" + read + "]";
         List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryInt(suppliers, read, DataTypes.INTEGER, i -> i, Integer.MIN_VALUE, Integer.MAX_VALUE, List.of());
+        TestCaseSupplier.forUnaryInt(suppliers, read, DataType.INTEGER, i -> i, Integer.MIN_VALUE, Integer.MAX_VALUE, List.of());
 
-        TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataTypes.INTEGER, b -> b ? 1 : 0, List.of());
+        TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataType.INTEGER, b -> b ? 1 : 0, List.of());
 
         // datetimes that fall within Integer's range
         TestCaseSupplier.unary(
             suppliers,
             evaluatorName.apply("Long"),
             dateCases(0, Integer.MAX_VALUE),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             l -> ((Long) l).intValue(),
             List.of()
         );
@@ -56,7 +56,7 @@ public static Iterable<Object[]> parameters() {
             suppliers,
             evaluatorName.apply("Long"),
             dateCases(Integer.MAX_VALUE + 1L, Long.MAX_VALUE),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             l -> null,
             l -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -67,7 +67,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryStrings(
             suppliers,
             evaluatorName.apply("String"),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -80,7 +80,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             evaluatorName.apply("Double"),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             d -> safeToInt(Math.round(d)),
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
@@ -90,7 +90,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             evaluatorName.apply("Double"),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             d -> null,
             Double.NEGATIVE_INFINITY,
             Integer.MIN_VALUE - 1d,
@@ -103,7 +103,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             evaluatorName.apply("Double"),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             d -> null,
             Integer.MAX_VALUE + 1d,
             Double.POSITIVE_INFINITY,
@@ -117,7 +117,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             evaluatorName.apply("UnsignedLong"),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             BigInteger::intValue,
             BigInteger.ZERO,
             BigInteger.valueOf(Integer.MAX_VALUE),
@@ -127,7 +127,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             evaluatorName.apply("UnsignedLong"),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             ul -> null,
             BigInteger.valueOf(Integer.MAX_VALUE).add(BigInteger.ONE),
             UNSIGNED_LONG_MAX,
@@ -142,7 +142,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             evaluatorName.apply("Long"),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             l -> (int) l,
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
@@ -152,7 +152,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             evaluatorName.apply("Long"),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             l -> null,
             Long.MIN_VALUE,
             Integer.MIN_VALUE - 1L,
@@ -166,7 +166,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             evaluatorName.apply("Long"),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             l -> null,
             Integer.MAX_VALUE + 1L,
             Long.MAX_VALUE,
@@ -186,11 +186,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             bytesRef -> Integer.valueOf(((BytesRef) bytesRef).utf8ToString()),
             List.of()
         );
@@ -204,11 +204,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             bytesRef -> safeToInt(Math.round(Double.parseDouble(((BytesRef) bytesRef).utf8ToString()))),
             List.of()
         );
@@ -222,11 +222,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -245,11 +245,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -262,8 +262,8 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.unary(
             suppliers,
             "Attribute[channel=0]",
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, DataTypes.COUNTER_INTEGER)),
-            DataTypes.INTEGER,
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, DataType.COUNTER_INTEGER)),
+            DataType.INTEGER,
             l -> l,
             List.of()
         );
@@ -279,13 +279,13 @@ protected Expression build(Source source, List<Expression> args) {
     private static List<TestCaseSupplier.TypedDataSupplier> dateCases(long min, long max) {
         List<TestCaseSupplier.TypedDataSupplier> dataSuppliers = new ArrayList<>(2);
         if (min == 0L) {
-            dataSuppliers.add(new TestCaseSupplier.TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataTypes.DATETIME));
+            dataSuppliers.add(new TestCaseSupplier.TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataType.DATETIME));
         }
         if (max <= Integer.MAX_VALUE) {
-            dataSuppliers.add(new TestCaseSupplier.TypedDataSupplier("<1970-01-25T20:31:23.647Z>", () -> 2147483647L, DataTypes.DATETIME));
+            dataSuppliers.add(new TestCaseSupplier.TypedDataSupplier("<1970-01-25T20:31:23.647Z>", () -> 2147483647L, DataType.DATETIME));
         }
         dataSuppliers.add(
-            new TestCaseSupplier.TypedDataSupplier("<date>", () -> ESTestCase.randomLongBetween(min, max), DataTypes.DATETIME)
+            new TestCaseSupplier.TypedDataSupplier("<date>", () -> ESTestCase.randomLongBetween(min, max), DataType.DATETIME)
         );
         return dataSuppliers;
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
index adb2d68cf526a..92b0bb192e2aa 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -37,17 +37,17 @@ public static Iterable<Object[]> parameters() {
         Function<String, String> evaluatorName = s -> "ToLongFrom" + s + "Evaluator[field=" + read + "]";
         List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        TestCaseSupplier.forUnaryLong(suppliers, read, DataTypes.LONG, l -> l, Long.MIN_VALUE, Long.MAX_VALUE, List.of());
+        TestCaseSupplier.forUnaryLong(suppliers, read, DataType.LONG, l -> l, Long.MIN_VALUE, Long.MAX_VALUE, List.of());
 
-        TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataTypes.LONG, b -> b ? 1L : 0L, List.of());
+        TestCaseSupplier.forUnaryBoolean(suppliers, evaluatorName.apply("Boolean"), DataType.LONG, b -> b ? 1L : 0L, List.of());
 
         // datetimes
-        TestCaseSupplier.forUnaryDatetime(suppliers, read, DataTypes.LONG, Instant::toEpochMilli, List.of());
+        TestCaseSupplier.forUnaryDatetime(suppliers, read, DataType.LONG, Instant::toEpochMilli, List.of());
         // random strings that don't look like a long
         TestCaseSupplier.forUnaryStrings(
             suppliers,
             evaluatorName.apply("String"),
-            DataTypes.LONG,
+            DataType.LONG,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -60,7 +60,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             evaluatorName.apply("Double"),
-            DataTypes.LONG,
+            DataType.LONG,
             Math::round,
             Long.MIN_VALUE,
             Long.MAX_VALUE,
@@ -70,7 +70,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             evaluatorName.apply("Double"),
-            DataTypes.LONG,
+            DataType.LONG,
             d -> null,
             Double.NEGATIVE_INFINITY,
             Long.MIN_VALUE - 1d,
@@ -83,7 +83,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             evaluatorName.apply("Double"),
-            DataTypes.LONG,
+            DataType.LONG,
             d -> null,
             Long.MAX_VALUE + 1d,
             Double.POSITIVE_INFINITY,
@@ -97,7 +97,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             evaluatorName.apply("UnsignedLong"),
-            DataTypes.LONG,
+            DataType.LONG,
             BigInteger::longValue,
             BigInteger.ZERO,
             BigInteger.valueOf(Long.MAX_VALUE),
@@ -106,7 +106,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             evaluatorName.apply("UnsignedLong"),
-            DataTypes.LONG,
+            DataType.LONG,
             ul -> null,
             BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE),
             UNSIGNED_LONG_MAX,
@@ -121,7 +121,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             evaluatorName.apply("Int"),
-            DataTypes.LONG,
+            DataType.LONG,
             l -> (long) l,
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
@@ -138,11 +138,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.LONG,
+            DataType.LONG,
             bytesRef -> Long.valueOf(((BytesRef) bytesRef).utf8ToString()),
             List.of()
         );
@@ -156,11 +156,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.LONG,
+            DataType.LONG,
             bytesRef -> Math.round(Double.parseDouble(((BytesRef) bytesRef).utf8ToString())),
             List.of()
         );
@@ -174,11 +174,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.LONG,
+            DataType.LONG,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -197,11 +197,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.LONG,
+            DataType.LONG,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -214,16 +214,16 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.unary(
             suppliers,
             "Attribute[channel=0]",
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomNonNegativeLong, DataTypes.COUNTER_LONG)),
-            DataTypes.LONG,
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomNonNegativeLong, DataType.COUNTER_LONG)),
+            DataType.LONG,
             l -> l,
             List.of()
         );
         TestCaseSupplier.unary(
             suppliers,
             evaluatorName.apply("Integer"),
-            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, DataTypes.COUNTER_INTEGER)),
-            DataTypes.LONG,
+            List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, DataType.COUNTER_INTEGER)),
+            DataType.LONG,
             l -> ((Integer) l).longValue(),
             List.of()
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java
index f2c3378a5408a..67951b46d03b5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -36,7 +36,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             evaluatorName.apply("ToDoubleFromIntEvaluator"),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::toRadians,
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
@@ -45,7 +45,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             evaluatorName.apply("ToDoubleFromLongEvaluator"),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::toRadians,
             Long.MIN_VALUE,
             Long.MAX_VALUE,
@@ -54,7 +54,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             evaluatorName.apply("ToDoubleFromUnsignedLongEvaluator"),
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             ul -> Math.toRadians(ul.doubleValue()),
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
@@ -63,7 +63,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "ToRadiansEvaluator[field=Attribute[channel=0]]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::toRadians,
             Double.NEGATIVE_INFINITY,
             Double.POSITIVE_INFINITY,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java
index 6e41930fce2e0..511df557ff842 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -40,7 +40,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "ToStringFromIntEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             i -> new BytesRef(Integer.toString(i)),
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
@@ -49,7 +49,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             "ToStringFromLongEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             l -> new BytesRef(Long.toString(l)),
             Long.MIN_VALUE,
             Long.MAX_VALUE,
@@ -58,7 +58,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "ToStringFromUnsignedLongEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             ul -> new BytesRef(ul.toString()),
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
@@ -67,7 +67,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "ToStringFromDoubleEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             d -> new BytesRef(Double.toString(d)),
             Double.NEGATIVE_INFINITY,
             Double.POSITIVE_INFINITY,
@@ -76,57 +76,57 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryBoolean(
             suppliers,
             "ToStringFromBooleanEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             b -> new BytesRef(b.toString()),
             List.of()
         );
         TestCaseSupplier.forUnaryDatetime(
             suppliers,
             "ToStringFromDatetimeEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             i -> new BytesRef(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(i.toEpochMilli())),
             List.of()
         );
         TestCaseSupplier.forUnaryGeoPoint(
             suppliers,
             "ToStringFromGeoPointEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             wkb -> new BytesRef(GEO.wkbToWkt(wkb)),
             List.of()
         );
         TestCaseSupplier.forUnaryCartesianPoint(
             suppliers,
             "ToStringFromCartesianPointEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             wkb -> new BytesRef(CARTESIAN.wkbToWkt(wkb)),
             List.of()
         );
         TestCaseSupplier.forUnaryGeoShape(
             suppliers,
             "ToStringFromGeoShapeEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             wkb -> new BytesRef(GEO.wkbToWkt(wkb)),
             List.of()
         );
         TestCaseSupplier.forUnaryCartesianShape(
             suppliers,
             "ToStringFromCartesianShapeEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             wkb -> new BytesRef(CARTESIAN.wkbToWkt(wkb)),
             List.of()
         );
         TestCaseSupplier.forUnaryIp(
             suppliers,
             "ToStringFromIPEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             ip -> new BytesRef(DocValueFormat.IP.format(ip)),
             List.of()
         );
-        TestCaseSupplier.forUnaryStrings(suppliers, read, DataTypes.KEYWORD, bytesRef -> bytesRef, List.of());
+        TestCaseSupplier.forUnaryStrings(suppliers, read, DataType.KEYWORD, bytesRef -> bytesRef, List.of());
         TestCaseSupplier.forUnaryVersion(
             suppliers,
             "ToStringFromVersionEvaluator[field=" + read + "]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             v -> new BytesRef(v.toString()),
             List.of()
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java
index e7e27e34e0a70..4182f99d316fc 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -42,7 +42,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             read,
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             n -> n,
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
@@ -52,7 +52,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryBoolean(
             suppliers,
             evaluatorName.apply("Boolean"),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             b -> b ? BigInteger.ONE : BigInteger.ZERO,
             List.of()
         );
@@ -61,12 +61,12 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDatetime(
             suppliers,
             evaluatorName.apply("Long"),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             instant -> BigInteger.valueOf(instant.toEpochMilli()),
             List.of()
         );
         // random strings that don't look like an unsigned_long
-        TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("String"), DataTypes.UNSIGNED_LONG, bytesRef -> null, bytesRef -> {
+        TestCaseSupplier.forUnaryStrings(suppliers, evaluatorName.apply("String"), DataType.UNSIGNED_LONG, bytesRef -> null, bytesRef -> {
             // BigDecimal, used to parse unsigned_longs will throw NFEs with different messages depending on empty string, first
             // non-number character after a number-looking like prefix, or string starting with "e", maybe others -- safer to take
             // this shortcut here.
@@ -80,7 +80,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             evaluatorName.apply("Double"),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             d -> BigDecimal.valueOf(d).toBigInteger(), // note: not: new BigDecimal(d).toBigInteger
             0d,
             UNSIGNED_LONG_MAX_AS_DOUBLE,
@@ -90,7 +90,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             evaluatorName.apply("Double"),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             d -> null,
             Double.NEGATIVE_INFINITY,
             -1d,
@@ -103,7 +103,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             evaluatorName.apply("Double"),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             d -> null,
             UNSIGNED_LONG_MAX_AS_DOUBLE + 10e5,
             Double.POSITIVE_INFINITY,
@@ -117,7 +117,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             evaluatorName.apply("Long"),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             BigInteger::valueOf,
             0L,
             Long.MAX_VALUE,
@@ -127,7 +127,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             evaluatorName.apply("Long"),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             unused -> null,
             Long.MIN_VALUE,
             -1L,
@@ -141,7 +141,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             evaluatorName.apply("Int"),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             BigInteger::valueOf,
             0,
             Integer.MAX_VALUE,
@@ -151,7 +151,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             evaluatorName.apply("Int"),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             unused -> null,
             Integer.MIN_VALUE,
             -1,
@@ -171,11 +171,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             bytesRef -> safeToUnsignedLong(((BytesRef) bytesRef).utf8ToString()),
             List.of()
         );
@@ -189,11 +189,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             bytesRef -> safeToUnsignedLong(((BytesRef) bytesRef).utf8ToString()),
             List.of()
         );
@@ -207,11 +207,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
@@ -230,11 +230,11 @@ public static Iterable<Object[]> parameters() {
                     tds -> new TestCaseSupplier.TypedDataSupplier(
                         tds.name() + "as string",
                         () -> new BytesRef(tds.supplier().get().toString()),
-                        DataTypes.KEYWORD
+                        DataType.KEYWORD
                     )
                 )
                 .toList(),
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             bytesRef -> null,
             bytesRef -> List.of(
                 "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.",
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
index 16eac2a6f1712..a397de64aeea8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.versionfield.Version;
@@ -36,7 +35,7 @@ public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
 
         // Converting and IP to an IP doesn't change anything. Everything should succeed.
-        TestCaseSupplier.forUnaryVersion(suppliers, read, DataTypes.VERSION, Version::toBytesRef, List.of());
+        TestCaseSupplier.forUnaryVersion(suppliers, read, DataType.VERSION, Version::toBytesRef, List.of());
 
         // None of the random strings ever look like versions so they should all become "invalid" versions:
         // https://github.com/elastic/elasticsearch/issues/98989
@@ -44,7 +43,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryStrings(
             suppliers,
             stringEvaluator,
-            DataTypes.VERSION,
+            DataType.VERSION,
             bytesRef -> new Version(bytesRef.utf8ToString()).toBytesRef(),
             List.of()
         );
@@ -55,7 +54,7 @@ public static Iterable<Object[]> parameters() {
                 suppliers,
                 read,
                 TestCaseSupplier.versionCases(inputType.typeName() + " "),
-                DataTypes.VERSION,
+                DataType.VERSION,
                 bytesRef -> new Version((BytesRef) bytesRef).toBytesRef(),
                 List.of()
             );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java
index f2f5846bf0901..89cfda5c4bce5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -39,66 +39,66 @@ public static Iterable<Object[]> parameters() {
             List.of(
                 new TestCaseSupplier(
                     "Date Diff In Seconds - OK",
-                    List.of(DataTypes.KEYWORD, DataTypes.DATETIME, DataTypes.DATETIME),
+                    List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME),
                     () -> new TestCaseSupplier.TestCase(
                         List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataTypes.KEYWORD, "unit"),
-                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataTypes.DATETIME, "startTimestamp"),
-                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataTypes.DATETIME, "endTimestamp")
+                            new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.KEYWORD, "unit"),
+                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
+                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
                         ),
                         "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], "
                             + "endTimestamp=Attribute[channel=2]]",
-                        DataTypes.INTEGER,
+                        DataType.INTEGER,
                         equalTo(88170)
                     )
                 ),
                 new TestCaseSupplier(
                     "Date Diff In Seconds with text- OK",
-                    List.of(DataTypes.TEXT, DataTypes.DATETIME, DataTypes.DATETIME),
+                    List.of(DataType.TEXT, DataType.DATETIME, DataType.DATETIME),
                     () -> new TestCaseSupplier.TestCase(
                         List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataTypes.TEXT, "unit"),
-                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataTypes.DATETIME, "startTimestamp"),
-                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataTypes.DATETIME, "endTimestamp")
+                            new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.TEXT, "unit"),
+                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
+                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
                         ),
                         "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], "
                             + "endTimestamp=Attribute[channel=2]]",
-                        DataTypes.INTEGER,
+                        DataType.INTEGER,
                         equalTo(88170)
                     )
                 ),
                 new TestCaseSupplier(
                     "Date Diff Error Type unit",
-                    List.of(DataTypes.INTEGER, DataTypes.DATETIME, DataTypes.DATETIME),
+                    List.of(DataType.INTEGER, DataType.DATETIME, DataType.DATETIME),
                     () -> TestCaseSupplier.TestCase.typeError(
                         List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataTypes.INTEGER, "unit"),
-                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataTypes.DATETIME, "startTimestamp"),
-                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataTypes.DATETIME, "endTimestamp")
+                            new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.INTEGER, "unit"),
+                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
+                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
                         ),
                         "first argument of [] must be [string], found value [unit] type [integer]"
                     )
                 ),
                 new TestCaseSupplier(
                     "Date Diff Error Type startTimestamp",
-                    List.of(DataTypes.TEXT, DataTypes.INTEGER, DataTypes.DATETIME),
+                    List.of(DataType.TEXT, DataType.INTEGER, DataType.DATETIME),
                     () -> TestCaseSupplier.TestCase.typeError(
                         List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataTypes.TEXT, "unit"),
-                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataTypes.INTEGER, "startTimestamp"),
-                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataTypes.DATETIME, "endTimestamp")
+                            new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"),
+                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.INTEGER, "startTimestamp"),
+                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
                         ),
                         "second argument of [] must be [datetime], found value [startTimestamp] type [integer]"
                     )
                 ),
                 new TestCaseSupplier(
                     "Date Diff Error Type endTimestamp",
-                    List.of(DataTypes.TEXT, DataTypes.DATETIME, DataTypes.INTEGER),
+                    List.of(DataType.TEXT, DataType.DATETIME, DataType.INTEGER),
                     () -> TestCaseSupplier.TestCase.typeError(
                         List.of(
-                            new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataTypes.TEXT, "unit"),
-                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataTypes.DATETIME, "startTimestamp"),
-                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataTypes.INTEGER, "endTimestamp")
+                            new TestCaseSupplier.TypedData(new BytesRef("minutes"), DataType.TEXT, "unit"),
+                            new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
+                            new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.INTEGER, "endTimestamp")
                         ),
                         "third argument of [] must be [datetime], found value [endTimestamp] type [integer]"
                     )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
index e2dc759b96832..221f3fd51a545 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
 import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
@@ -45,39 +45,39 @@ public static Iterable<Object[]> parameters() {
                     true,
                     List.of(
                         new TestCaseSupplier(
-                            List.of(DataTypes.KEYWORD, DataTypes.DATETIME),
+                            List.of(DataType.KEYWORD, DataType.DATETIME),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("YeAr"), DataTypes.KEYWORD, "chrono"),
-                                    new TestCaseSupplier.TypedData(1687944333000L, DataTypes.DATETIME, "date")
+                                    new TestCaseSupplier.TypedData(new BytesRef("YeAr"), DataType.KEYWORD, "chrono"),
+                                    new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "date")
                                 ),
                                 "DateExtractEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]",
-                                DataTypes.LONG,
+                                DataType.LONG,
                                 equalTo(2023L)
                             )
                         ),
                         new TestCaseSupplier(
-                            List.of(DataTypes.TEXT, DataTypes.DATETIME),
+                            List.of(DataType.TEXT, DataType.DATETIME),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("YeAr"), DataTypes.TEXT, "chrono"),
-                                    new TestCaseSupplier.TypedData(1687944333000L, DataTypes.DATETIME, "date")
+                                    new TestCaseSupplier.TypedData(new BytesRef("YeAr"), DataType.TEXT, "chrono"),
+                                    new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "date")
                                 ),
                                 "DateExtractEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]",
-                                DataTypes.LONG,
+                                DataType.LONG,
                                 equalTo(2023L)
                             )
                         ),
                         new TestCaseSupplier(
-                            List.of(DataTypes.KEYWORD, DataTypes.DATETIME),
+                            List.of(DataType.KEYWORD, DataType.DATETIME),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("not a unit"), DataTypes.KEYWORD, "chrono"),
-                                    new TestCaseSupplier.TypedData(0L, DataTypes.DATETIME, "date")
+                                    new TestCaseSupplier.TypedData(new BytesRef("not a unit"), DataType.KEYWORD, "chrono"),
+                                    new TestCaseSupplier.TypedData(0L, DataType.DATETIME, "date")
 
                                 ),
                                 "DateExtractEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]",
-                                DataTypes.LONG,
+                                DataType.LONG,
                                 is(nullValue())
                             ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
                                 .withWarning(
@@ -98,8 +98,8 @@ public void testAllChronoFields() {
         for (ChronoField value : ChronoField.values()) {
             DateExtract instance = new DateExtract(
                 Source.EMPTY,
-                new Literal(Source.EMPTY, new BytesRef(value.name()), DataTypes.KEYWORD),
-                new Literal(Source.EMPTY, epochMilli, DataTypes.DATETIME),
+                new Literal(Source.EMPTY, new BytesRef(value.name()), DataType.KEYWORD),
+                new Literal(Source.EMPTY, epochMilli, DataType.DATETIME),
                 EsqlTestUtils.TEST_CFG
             );
 
@@ -119,8 +119,8 @@ public void testInvalidChrono() {
             () -> evaluator(
                 new DateExtract(
                     Source.EMPTY,
-                    new Literal(Source.EMPTY, new BytesRef(chrono), DataTypes.KEYWORD),
-                    field("str", DataTypes.DATETIME),
+                    new Literal(Source.EMPTY, new BytesRef(chrono), DataType.KEYWORD),
+                    field("str", DataType.DATETIME),
                     null
                 )
             ).get(driverContext)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java
index 26f0e88ff2fce..6e1b5caa710e1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
 import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
@@ -37,26 +37,26 @@ public static Iterable<Object[]> parameters() {
                     true,
                     List.of(
                         new TestCaseSupplier(
-                            List.of(DataTypes.KEYWORD, DataTypes.DATETIME),
+                            List.of(DataType.KEYWORD, DataType.DATETIME),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy"), DataTypes.KEYWORD, "formatter"),
-                                    new TestCaseSupplier.TypedData(1687944333000L, DataTypes.DATETIME, "val")
+                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy"), DataType.KEYWORD, "formatter"),
+                                    new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "val")
                                 ),
                                 "DateFormatEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], locale=en_US]",
-                                DataTypes.KEYWORD,
+                                DataType.KEYWORD,
                                 equalTo(BytesRefs.toBytesRef("2023"))
                             )
                         ),
                         new TestCaseSupplier(
-                            List.of(DataTypes.TEXT, DataTypes.DATETIME),
+                            List.of(DataType.TEXT, DataType.DATETIME),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy"), DataTypes.TEXT, "formatter"),
-                                    new TestCaseSupplier.TypedData(1687944333000L, DataTypes.DATETIME, "val")
+                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy"), DataType.TEXT, "formatter"),
+                                    new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "val")
                                 ),
                                 "DateFormatEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], locale=en_US]",
-                                DataTypes.KEYWORD,
+                                DataType.KEYWORD,
                                 equalTo(BytesRefs.toBytesRef("2023"))
                             )
                         )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
index 19c5ca0de72f5..8906994c6d7eb 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -42,66 +42,66 @@ public static Iterable<Object[]> parameters() {
                     List.of(
                         new TestCaseSupplier(
                             "Basic Case",
-                            List.of(DataTypes.KEYWORD, DataTypes.KEYWORD),
+                            List.of(DataType.KEYWORD, DataType.KEYWORD),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataTypes.KEYWORD, "first"),
-                                    new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataTypes.KEYWORD, "second")
+                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
+                                    new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
                                 ),
                                 "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
-                                DataTypes.DATETIME,
+                                DataType.DATETIME,
                                 equalTo(1683244800000L)
                             )
                         ),
                         new TestCaseSupplier(
                             "With Text",
-                            List.of(DataTypes.KEYWORD, DataTypes.TEXT),
+                            List.of(DataType.KEYWORD, DataType.TEXT),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataTypes.KEYWORD, "first"),
-                                    new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataTypes.TEXT, "second")
+                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
+                                    new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
                                 ),
                                 "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
-                                DataTypes.DATETIME,
+                                DataType.DATETIME,
                                 equalTo(1683244800000L)
                             )
                         ),
                         new TestCaseSupplier(
                             "With Both Text",
-                            List.of(DataTypes.TEXT, DataTypes.TEXT),
+                            List.of(DataType.TEXT, DataType.TEXT),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataTypes.TEXT, "first"),
-                                    new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataTypes.TEXT, "second")
+                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
+                                    new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
                                 ),
                                 "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
-                                DataTypes.DATETIME,
+                                DataType.DATETIME,
                                 equalTo(1683244800000L)
                             )
                         ),
                         new TestCaseSupplier(
                             "With keyword",
-                            List.of(DataTypes.TEXT, DataTypes.KEYWORD),
+                            List.of(DataType.TEXT, DataType.KEYWORD),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataTypes.TEXT, "first"),
-                                    new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataTypes.KEYWORD, "second")
+                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
+                                    new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
                                 ),
                                 "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
-                                DataTypes.DATETIME,
+                                DataType.DATETIME,
                                 equalTo(1683244800000L)
                             )
                         ),
                         new TestCaseSupplier(
-                            List.of(DataTypes.KEYWORD, DataTypes.KEYWORD),
+                            List.of(DataType.KEYWORD, DataType.KEYWORD),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("not a format"), DataTypes.KEYWORD, "first"),
-                                    new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataTypes.KEYWORD, "second")
+                                    new TestCaseSupplier.TypedData(new BytesRef("not a format"), DataType.KEYWORD, "first"),
+                                    new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
 
                                 ),
                                 "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
-                                DataTypes.DATETIME,
+                                DataType.DATETIME,
                                 is(nullValue())
                             ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
                                 .withWarning(
@@ -114,15 +114,15 @@ public static Iterable<Object[]> parameters() {
                                 )
                         ),
                         new TestCaseSupplier(
-                            List.of(DataTypes.KEYWORD, DataTypes.KEYWORD),
+                            List.of(DataType.KEYWORD, DataType.KEYWORD),
                             () -> new TestCaseSupplier.TestCase(
                                 List.of(
-                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataTypes.KEYWORD, "first"),
-                                    new TestCaseSupplier.TypedData(new BytesRef("not a date"), DataTypes.KEYWORD, "second")
+                                    new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
+                                    new TestCaseSupplier.TypedData(new BytesRef("not a date"), DataType.KEYWORD, "second")
 
                                 ),
                                 "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
-                                DataTypes.DATETIME,
+                                DataType.DATETIME,
                                 is(nullValue())
                             ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
                                 .withWarning(
@@ -144,8 +144,8 @@ public void testInvalidPattern() {
             () -> evaluator(
                 new DateParse(
                     Source.EMPTY,
-                    new Literal(Source.EMPTY, new BytesRef(pattern), DataTypes.KEYWORD),
-                    field("str", DataTypes.KEYWORD)
+                    new Literal(Source.EMPTY, new BytesRef(pattern), DataType.KEYWORD),
+                    field("str", DataType.KEYWORD)
                 )
             ).get(driverContext)
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
index f33f8d4ba1041..b627d7cd88908 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncTests.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -139,14 +139,14 @@ public void testDateTruncFunction() {
 
     private static TestCaseSupplier ofDatePeriod(Period period, long value, String expectedDate) {
         return new TestCaseSupplier(
-            List.of(DataTypes.DATE_PERIOD, DataTypes.DATETIME),
+            List.of(DataType.DATE_PERIOD, DataType.DATETIME),
             () -> new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(period, DataTypes.DATE_PERIOD, "interval"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.DATETIME, "date")
+                    new TestCaseSupplier.TypedData(period, DataType.DATE_PERIOD, "interval"),
+                    new TestCaseSupplier.TypedData(value, DataType.DATETIME, "date")
                 ),
                 "DateTruncEvaluator[date=Attribute[channel=1], interval=Attribute[channel=0]]",
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 equalTo(toMillis(expectedDate))
             )
         );
@@ -154,21 +154,21 @@ private static TestCaseSupplier ofDatePeriod(Period period, long value, String e
 
     private static TestCaseSupplier ofDuration(Duration duration, long value, String expectedDate) {
         return new TestCaseSupplier(
-            List.of(DataTypes.TIME_DURATION, DataTypes.DATETIME),
+            List.of(DataType.TIME_DURATION, DataType.DATETIME),
             () -> new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(duration, DataTypes.TIME_DURATION, "interval"),
-                    new TestCaseSupplier.TypedData(value, DataTypes.DATETIME, "date")
+                    new TestCaseSupplier.TypedData(duration, DataType.TIME_DURATION, "interval"),
+                    new TestCaseSupplier.TypedData(value, DataType.DATETIME, "date")
                 ),
                 "DateTruncEvaluator[date=Attribute[channel=1], interval=Attribute[channel=0]]",
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 equalTo(toMillis(expectedDate))
             )
         );
     }
 
     private static TestCaseSupplier randomSecond() {
-        return new TestCaseSupplier("random second", List.of(DataTypes.TIME_DURATION, DataTypes.DATETIME), () -> {
+        return new TestCaseSupplier("random second", List.of(DataType.TIME_DURATION, DataType.DATETIME), () -> {
             String dateFragment = randomIntBetween(2000, 2050)
                 + "-"
                 + pad(randomIntBetween(1, 12))
@@ -182,11 +182,11 @@ private static TestCaseSupplier randomSecond() {
                 + pad(randomIntBetween(0, 59));
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(Duration.ofSeconds(1), DataTypes.TIME_DURATION, "interval"),
-                    new TestCaseSupplier.TypedData(toMillis(dateFragment + ".38Z"), DataTypes.DATETIME, "date")
+                    new TestCaseSupplier.TypedData(Duration.ofSeconds(1), DataType.TIME_DURATION, "interval"),
+                    new TestCaseSupplier.TypedData(toMillis(dateFragment + ".38Z"), DataType.DATETIME, "date")
                 ),
                 "DateTruncEvaluator[date=Attribute[channel=1], interval=Attribute[channel=0]]",
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 equalTo(toMillis(dateFragment + ".00Z"))
             );
         });
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java
index c86c5938dc4ba..2aaca179b2bc4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -35,50 +35,50 @@ public static Iterable<Object[]> parameters() {
 
         var suppliers = List.of(
             new TestCaseSupplier(
-                List.of(DataTypes.IP, DataTypes.KEYWORD),
+                List.of(DataType.IP, DataType.KEYWORD),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.0.10"), DataTypes.IP, "ip"),
-                        new TestCaseSupplier.TypedData(new BytesRef("192.168.0.0/16"), DataTypes.KEYWORD, "cidrs")
+                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.0.10"), DataType.IP, "ip"),
+                        new TestCaseSupplier.TypedData(new BytesRef("192.168.0.0/16"), DataType.KEYWORD, "cidrs")
                     ),
                     "CIDRMatchEvaluator[ip=Attribute[channel=0], cidrs=[Attribute[channel=1]]]",
-                    DataTypes.BOOLEAN,
+                    DataType.BOOLEAN,
                     equalTo(true)
                 )
             ),
             new TestCaseSupplier(
-                List.of(DataTypes.IP, DataTypes.TEXT),
+                List.of(DataType.IP, DataType.TEXT),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.0.10"), DataTypes.IP, "ip"),
-                        new TestCaseSupplier.TypedData(new BytesRef("192.168.0.0/16"), DataTypes.TEXT, "cidrs")
+                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.0.10"), DataType.IP, "ip"),
+                        new TestCaseSupplier.TypedData(new BytesRef("192.168.0.0/16"), DataType.TEXT, "cidrs")
                     ),
                     "CIDRMatchEvaluator[ip=Attribute[channel=0], cidrs=[Attribute[channel=1]]]",
-                    DataTypes.BOOLEAN,
+                    DataType.BOOLEAN,
                     equalTo(true)
                 )
             ),
             new TestCaseSupplier(
-                List.of(DataTypes.IP, DataTypes.KEYWORD),
+                List.of(DataType.IP, DataType.KEYWORD),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.0.10"), DataTypes.IP, "ip"),
-                        new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/16"), DataTypes.KEYWORD, "cidrs")
+                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.0.10"), DataType.IP, "ip"),
+                        new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/16"), DataType.KEYWORD, "cidrs")
                     ),
                     "CIDRMatchEvaluator[ip=Attribute[channel=0], cidrs=[Attribute[channel=1]]]",
-                    DataTypes.BOOLEAN,
+                    DataType.BOOLEAN,
                     equalTo(false)
                 )
             ),
             new TestCaseSupplier(
-                List.of(DataTypes.IP, DataTypes.TEXT),
+                List.of(DataType.IP, DataType.TEXT),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.0.10"), DataTypes.IP, "ip"),
-                        new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/16"), DataTypes.TEXT, "cidrs")
+                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("192.168.0.10"), DataType.IP, "ip"),
+                        new TestCaseSupplier.TypedData(new BytesRef("10.0.0.0/16"), DataType.TEXT, "cidrs")
                     ),
                     "CIDRMatchEvaluator[ip=Attribute[channel=0], cidrs=[Attribute[channel=1]]]",
-                    DataTypes.BOOLEAN,
+                    DataType.BOOLEAN,
                     equalTo(false)
                 )
             )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
index e46eaea849bb5..063a057134d7e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
@@ -35,19 +35,19 @@ public static Iterable<Object[]> parameters() {
         var suppliers = List.of(
             // V4
             new TestCaseSupplier(
-                List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER),
+                List.of(DataType.IP, DataType.INTEGER, DataType.INTEGER),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("1.2.3.4"), DataTypes.IP, "ip"),
-                        new TestCaseSupplier.TypedData(24, DataTypes.INTEGER, "prefixLengthV4"),
-                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 128), DataTypes.INTEGER, "prefixLengthV6")
+                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("1.2.3.4"), DataType.IP, "ip"),
+                        new TestCaseSupplier.TypedData(24, DataType.INTEGER, "prefixLengthV4"),
+                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 128), DataType.INTEGER, "prefixLengthV6")
                     ),
                     "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]",
-                    DataTypes.IP,
+                    DataType.IP,
                     equalTo(EsqlDataTypeConverter.stringToIP("1.2.3.0"))
                 )
             ),
-            new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+            new TestCaseSupplier(List.of(DataType.IP, DataType.INTEGER, DataType.INTEGER), () -> {
                 var randomIp = randomIp(true);
                 var randomPrefix = randomIntBetween(0, 32);
                 var cidrString = InetAddresses.toCidrString(randomIp, randomPrefix);
@@ -59,31 +59,31 @@ public static Iterable<Object[]> parameters() {
 
                 return new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(ipParameter, DataTypes.IP, "ip"),
-                        new TestCaseSupplier.TypedData(randomPrefix, DataTypes.INTEGER, "prefixLengthV4"),
-                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 128), DataTypes.INTEGER, "prefixLengthV6")
+                        new TestCaseSupplier.TypedData(ipParameter, DataType.IP, "ip"),
+                        new TestCaseSupplier.TypedData(randomPrefix, DataType.INTEGER, "prefixLengthV4"),
+                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 128), DataType.INTEGER, "prefixLengthV6")
                     ),
                     "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]",
-                    DataTypes.IP,
+                    DataType.IP,
                     equalTo(expectedPrefix)
                 );
             }),
 
             // V6
             new TestCaseSupplier(
-                List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER),
+                List.of(DataType.IP, DataType.INTEGER, DataType.INTEGER),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::ff"), DataTypes.IP, "ip"),
-                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 32), DataTypes.INTEGER, "prefixLengthV4"),
-                        new TestCaseSupplier.TypedData(127, DataTypes.INTEGER, "prefixLengthV6")
+                        new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::ff"), DataType.IP, "ip"),
+                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 32), DataType.INTEGER, "prefixLengthV4"),
+                        new TestCaseSupplier.TypedData(127, DataType.INTEGER, "prefixLengthV6")
                     ),
                     "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]",
-                    DataTypes.IP,
+                    DataType.IP,
                     equalTo(EsqlDataTypeConverter.stringToIP("::fe"))
                 )
             ),
-            new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+            new TestCaseSupplier(List.of(DataType.IP, DataType.INTEGER, DataType.INTEGER), () -> {
                 var randomIp = randomIp(false);
                 var randomPrefix = randomIntBetween(0, 128);
                 var cidrString = InetAddresses.toCidrString(randomIp, randomPrefix);
@@ -95,12 +95,12 @@ public static Iterable<Object[]> parameters() {
 
                 return new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(ipParameter, DataTypes.IP, "ip"),
-                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 32), DataTypes.INTEGER, "prefixLengthV4"),
-                        new TestCaseSupplier.TypedData(randomPrefix, DataTypes.INTEGER, "prefixLengthV6")
+                        new TestCaseSupplier.TypedData(ipParameter, DataType.IP, "ip"),
+                        new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 32), DataType.INTEGER, "prefixLengthV4"),
+                        new TestCaseSupplier.TypedData(randomPrefix, DataType.INTEGER, "prefixLengthV6")
                     ),
                     "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]",
-                    DataTypes.IP,
+                    DataType.IP,
                     equalTo(expectedPrefix)
                 );
             })
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java
index 3c1b85d51515a..5158fb9aad372 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
 
@@ -28,39 +27,39 @@ public class AbsTests extends AbstractScalarFunctionTestCase {
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.INTEGER), () -> {
             int arg = randomInt();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(arg, DataTypes.INTEGER, "arg")),
+                List.of(new TestCaseSupplier.TypedData(arg, DataType.INTEGER, "arg")),
                 "AbsIntEvaluator[fieldVal=Attribute[channel=0]]",
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 equalTo(Math.abs(arg))
             );
         }));
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "Attribute[channel=0]",
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             (n) -> n,
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
             List.of()
         );
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.LONG), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.LONG), () -> {
             long arg = randomLong();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(arg, DataTypes.LONG, "arg")),
+                List.of(new TestCaseSupplier.TypedData(arg, DataType.LONG, "arg")),
                 "AbsLongEvaluator[fieldVal=Attribute[channel=0]]",
-                DataTypes.LONG,
+                DataType.LONG,
                 equalTo(Math.abs(arg))
             );
         }));
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.DOUBLE), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.DOUBLE), () -> {
             double arg = randomDouble();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(arg, DataTypes.DOUBLE, "arg")),
+                List.of(new TestCaseSupplier.TypedData(arg, DataType.DOUBLE, "arg")),
                 "AbsDoubleEvaluator[fieldVal=Attribute[channel=0]]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 equalTo(Math.abs(arg))
             );
         }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java
index b9b672a2111c0..c4e614be94438 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket;
@@ -44,7 +43,7 @@ public static Iterable<Object[]> parameters() {
             suppliers,
             "fixed date with period",
             () -> DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00.00Z"),
-            DataTypes.DATE_PERIOD,
+            DataType.DATE_PERIOD,
             Period.ofYears(1),
             "[YEAR_OF_CENTURY in Z][fixed to midnight]"
         );
@@ -52,22 +51,22 @@ public static Iterable<Object[]> parameters() {
             suppliers,
             "fixed date with duration",
             () -> DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-02-17T09:00:00.00Z"),
-            DataTypes.TIME_DURATION,
+            DataType.TIME_DURATION,
             Duration.ofDays(1L),
             "[86400000 in Z][fixed]"
         );
-        numberCases(suppliers, "fixed long", DataTypes.LONG, () -> 100L);
-        numberCasesWithSpan(suppliers, "fixed long with span", DataTypes.LONG, () -> 100L);
-        numberCases(suppliers, "fixed int", DataTypes.INTEGER, () -> 100);
-        numberCasesWithSpan(suppliers, "fixed int with span", DataTypes.INTEGER, () -> 100);
-        numberCases(suppliers, "fixed double", DataTypes.DOUBLE, () -> 100.0);
-        numberCasesWithSpan(suppliers, "fixed double with span", DataTypes.DOUBLE, () -> 100.);
+        numberCases(suppliers, "fixed long", DataType.LONG, () -> 100L);
+        numberCasesWithSpan(suppliers, "fixed long with span", DataType.LONG, () -> 100L);
+        numberCases(suppliers, "fixed int", DataType.INTEGER, () -> 100);
+        numberCasesWithSpan(suppliers, "fixed int with span", DataType.INTEGER, () -> 100);
+        numberCases(suppliers, "fixed double", DataType.DOUBLE, () -> 100.0);
+        numberCasesWithSpan(suppliers, "fixed double with span", DataType.DOUBLE, () -> 100.);
         // TODO make errorsForCasesWithoutExamples do something sensible for 4+ parameters
         return parameterSuppliersFromTypedData(
             anyNullIsNull(
                 suppliers,
-                (nullPosition, nullValueDataType, original) -> nullPosition == 0 && nullValueDataType == DataTypes.NULL
-                    ? DataTypes.NULL
+                (nullPosition, nullValueDataType, original) -> nullPosition == 0 && nullValueDataType == DataType.NULL
+                    ? DataType.NULL
                     : original.expectedType(),
                 (nullPosition, nullData, original) -> nullPosition == 0 ? original : equalTo("LiteralsEvaluator[lit=null]")
             )
@@ -75,22 +74,22 @@ public static Iterable<Object[]> parameters() {
     }
 
     // TODO once we cast above the functions we can drop these
-    private static final DataType[] DATE_BOUNDS_TYPE = new DataType[] { DataTypes.DATETIME };
+    private static final DataType[] DATE_BOUNDS_TYPE = new DataType[] { DataType.DATETIME };
 
     private static void dateCases(List<TestCaseSupplier> suppliers, String name, LongSupplier date) {
         for (DataType fromType : DATE_BOUNDS_TYPE) {
             for (DataType toType : DATE_BOUNDS_TYPE) {
-                suppliers.add(new TestCaseSupplier(name, List.of(DataTypes.DATETIME, DataTypes.INTEGER, fromType, toType), () -> {
+                suppliers.add(new TestCaseSupplier(name, List.of(DataType.DATETIME, DataType.INTEGER, fromType, toType), () -> {
                     List<TestCaseSupplier.TypedData> args = new ArrayList<>();
-                    args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataTypes.DATETIME, "field"));
+                    args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataType.DATETIME, "field"));
                     // TODO more "from" and "to" and "buckets"
-                    args.add(new TestCaseSupplier.TypedData(50, DataTypes.INTEGER, "buckets").forceLiteral());
+                    args.add(new TestCaseSupplier.TypedData(50, DataType.INTEGER, "buckets").forceLiteral());
                     args.add(dateBound("from", fromType, "2023-02-01T00:00:00.00Z"));
                     args.add(dateBound("to", toType, "2023-03-01T09:00:00.00Z"));
                     return new TestCaseSupplier.TestCase(
                         args,
                         "DateTruncEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding[DAY_OF_MONTH in Z][fixed to midnight]]",
-                        DataTypes.DATETIME,
+                        DataType.DATETIME,
                         dateResultsMatcher(args)
                     );
                 }));
@@ -100,7 +99,7 @@ private static void dateCases(List<TestCaseSupplier> suppliers, String name, Lon
 
     private static TestCaseSupplier.TypedData dateBound(String name, DataType type, String date) {
         Object value;
-        if (type == DataTypes.DATETIME) {
+        if (type == DataType.DATETIME) {
             value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(date);
         } else {
             value = new BytesRef(date);
@@ -116,36 +115,36 @@ private static void dateCasesWithSpan(
         Object span,
         String spanStr
     ) {
-        suppliers.add(new TestCaseSupplier(name, List.of(DataTypes.DATETIME, spanType), () -> {
+        suppliers.add(new TestCaseSupplier(name, List.of(DataType.DATETIME, spanType), () -> {
             List<TestCaseSupplier.TypedData> args = new ArrayList<>();
-            args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataTypes.DATETIME, "field"));
+            args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataType.DATETIME, "field"));
             args.add(new TestCaseSupplier.TypedData(span, spanType, "buckets").forceLiteral());
             return new TestCaseSupplier.TestCase(
                 args,
                 "DateTruncEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding" + spanStr + "]",
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 dateResultsMatcher(args)
             );
         }));
     }
 
-    private static final DataType[] NUMBER_BOUNDS_TYPES = new DataType[] { DataTypes.INTEGER, DataTypes.LONG, DataTypes.DOUBLE };
+    private static final DataType[] NUMBER_BOUNDS_TYPES = new DataType[] { DataType.INTEGER, DataType.LONG, DataType.DOUBLE };
 
     private static void numberCases(List<TestCaseSupplier> suppliers, String name, DataType numberType, Supplier<Number> number) {
         for (DataType fromType : NUMBER_BOUNDS_TYPES) {
             for (DataType toType : NUMBER_BOUNDS_TYPES) {
-                suppliers.add(new TestCaseSupplier(name, List.of(numberType, DataTypes.INTEGER, fromType, toType), () -> {
+                suppliers.add(new TestCaseSupplier(name, List.of(numberType, DataType.INTEGER, fromType, toType), () -> {
                     List<TestCaseSupplier.TypedData> args = new ArrayList<>();
                     args.add(new TestCaseSupplier.TypedData(number.get(), "field"));
                     // TODO more "from" and "to" and "buckets"
-                    args.add(new TestCaseSupplier.TypedData(50, DataTypes.INTEGER, "buckets").forceLiteral());
+                    args.add(new TestCaseSupplier.TypedData(50, DataType.INTEGER, "buckets").forceLiteral());
                     args.add(numericBound("from", fromType, 0.0));
                     args.add(numericBound("to", toType, 1000.0));
                     // TODO more number types for "from" and "to"
                     String attr = "Attribute[channel=0]";
-                    if (numberType == DataTypes.INTEGER) {
+                    if (numberType == DataType.INTEGER) {
                         attr = "CastIntToDoubleEvaluator[v=" + attr + "]";
-                    } else if (numberType == DataTypes.LONG) {
+                    } else if (numberType == DataType.LONG) {
                         attr = "CastLongToDoubleEvaluator[v=" + attr + "]";
                     }
                     return new TestCaseSupplier.TestCase(
@@ -154,7 +153,7 @@ private static void numberCases(List<TestCaseSupplier> suppliers, String name, D
                             + attr
                             + ", "
                             + "rhs=LiteralsEvaluator[lit=50.0]]], rhs=LiteralsEvaluator[lit=50.0]]",
-                        DataTypes.DOUBLE,
+                        DataType.DOUBLE,
                         dateResultsMatcher(args)
                     );
                 }));
@@ -164,9 +163,9 @@ private static void numberCases(List<TestCaseSupplier> suppliers, String name, D
 
     private static TestCaseSupplier.TypedData numericBound(String name, DataType type, double value) {
         Number v;
-        if (type == DataTypes.INTEGER) {
+        if (type == DataType.INTEGER) {
             v = (int) value;
-        } else if (type == DataTypes.LONG) {
+        } else if (type == DataType.LONG) {
             v = (long) value;
         } else {
             v = value;
@@ -175,14 +174,14 @@ private static TestCaseSupplier.TypedData numericBound(String name, DataType typ
     }
 
     private static void numberCasesWithSpan(List<TestCaseSupplier> suppliers, String name, DataType numberType, Supplier<Number> number) {
-        suppliers.add(new TestCaseSupplier(name, List.of(numberType, DataTypes.DOUBLE), () -> {
+        suppliers.add(new TestCaseSupplier(name, List.of(numberType, DataType.DOUBLE), () -> {
             List<TestCaseSupplier.TypedData> args = new ArrayList<>();
             args.add(new TestCaseSupplier.TypedData(number.get(), "field"));
-            args.add(new TestCaseSupplier.TypedData(50., DataTypes.DOUBLE, "span").forceLiteral());
+            args.add(new TestCaseSupplier.TypedData(50., DataType.DOUBLE, "span").forceLiteral());
             String attr = "Attribute[channel=0]";
-            if (numberType == DataTypes.INTEGER) {
+            if (numberType == DataType.INTEGER) {
                 attr = "CastIntToDoubleEvaluator[v=" + attr + "]";
-            } else if (numberType == DataTypes.LONG) {
+            } else if (numberType == DataType.LONG) {
                 attr = "CastLongToDoubleEvaluator[v=" + attr + "]";
             }
             return new TestCaseSupplier.TestCase(
@@ -191,7 +190,7 @@ private static void numberCasesWithSpan(List<TestCaseSupplier> suppliers, String
                     + attr
                     + ", "
                     + "rhs=LiteralsEvaluator[lit=50.0]]], rhs=LiteralsEvaluator[lit=50.0]]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 dateResultsMatcher(args)
             );
         }));
@@ -203,7 +202,7 @@ private static TestCaseSupplier.TypedData keywordDateLiteral(String name, DataTy
     }
 
     private static Matcher<Object> dateResultsMatcher(List<TestCaseSupplier.TypedData> typedData) {
-        if (typedData.get(0).type() == DataTypes.DATETIME) {
+        if (typedData.get(0).type() == DataType.DATETIME) {
             long millis = ((Number) typedData.get(0).data()).longValue();
             return equalTo(Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build().prepareForUnknown().round(millis));
         }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java
index 8c9ff78bcdba8..14d6075f5cbe3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -37,7 +37,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "CbrtIntEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::cbrt,
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
@@ -46,7 +46,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             "CbrtLongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::cbrt,
             Long.MIN_VALUE,
             Long.MAX_VALUE,
@@ -55,7 +55,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "CbrtUnsignedLongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             ul -> Math.cbrt(unsignedLongToDouble(NumericUtils.asLongUnsigned(ul))),
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
@@ -64,7 +64,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "CbrtDoubleEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::cbrt,
             Double.MIN_VALUE,
             Double.MAX_VALUE,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java
index 70b8eafb88d24..f562ccbf0071b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
 
@@ -35,25 +34,25 @@ public static Iterable<Object[]> parameters() {
         suppliers.addAll(List.of(new TestCaseSupplier("large double value", () -> {
             double arg = 1 / randomDouble();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(arg, DataTypes.DOUBLE, "arg")),
+                List.of(new TestCaseSupplier.TypedData(arg, DataType.DOUBLE, "arg")),
                 "CeilDoubleEvaluator[val=Attribute[channel=0]]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 equalTo(Math.ceil(arg))
             );
         }), new TestCaseSupplier("integer value", () -> {
             int arg = randomInt();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(arg, DataTypes.INTEGER, "arg")),
+                List.of(new TestCaseSupplier.TypedData(arg, DataType.INTEGER, "arg")),
                 "Attribute[channel=0]",
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 equalTo(arg)
             );
         }), new TestCaseSupplier("long value", () -> {
             long arg = randomLong();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(arg, DataTypes.LONG, "arg")),
+                List.of(new TestCaseSupplier.TypedData(arg, DataType.LONG, "arg")),
                 "Attribute[channel=0]",
-                DataTypes.LONG,
+                DataType.LONG,
                 equalTo(arg)
             );
         })));
@@ -61,7 +60,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "Attribute[channel=0]",
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             (n) -> n,
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java
index 01d848ea9609b..8eb0b80fc21d7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -33,9 +33,9 @@ public ETests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupp
     public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("E Test", () -> {
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(1, DataTypes.INTEGER, "foo")),
+                List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")),
                 "LiteralsEvaluator[lit=2.718281828459045]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 equalTo(Math.E)
             );
         })));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java
index cb4fd8a403ed1..62c23369cc436 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -30,12 +30,12 @@ public FloorTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCase
     public static Iterable<Object[]> parameters() {
         String read = "Attribute[channel=0]";
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        TestCaseSupplier.forUnaryInt(suppliers, read, DataTypes.INTEGER, i -> i, Integer.MIN_VALUE, Integer.MAX_VALUE, List.of());
-        TestCaseSupplier.forUnaryLong(suppliers, read, DataTypes.LONG, l -> l, Long.MIN_VALUE, Long.MAX_VALUE, List.of());
+        TestCaseSupplier.forUnaryInt(suppliers, read, DataType.INTEGER, i -> i, Integer.MIN_VALUE, Integer.MAX_VALUE, List.of());
+        TestCaseSupplier.forUnaryLong(suppliers, read, DataType.LONG, l -> l, Long.MIN_VALUE, Long.MAX_VALUE, List.of());
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             read,
-            DataTypes.UNSIGNED_LONG,
+            DataType.UNSIGNED_LONG,
             ul -> ul,
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
@@ -44,7 +44,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "FloorDoubleEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::floor,
             Double.NEGATIVE_INFINITY,
             Double.POSITIVE_INFINITY,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java
index 95bc853c890c2..64329d7824b74 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -37,7 +37,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "Log10IntEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::log10,
             1,
             Integer.MAX_VALUE,
@@ -46,7 +46,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             "Log10LongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::log10,
             1L,
             Long.MAX_VALUE,
@@ -55,7 +55,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "Log10UnsignedLongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             ul -> Math.log10(ul == null ? null : unsignedLongToDouble(bigIntegerToUnsignedLong(ul))),
             BigInteger.ONE,
             UNSIGNED_LONG_MAX,
@@ -64,7 +64,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "Log10DoubleEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::log10,
             Double.MIN_VALUE,
             Double.POSITIVE_INFINITY,
@@ -78,7 +78,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "Log10IntEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             k -> null,
             Integer.MIN_VALUE,
             0,
@@ -90,7 +90,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             "Log10LongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             k -> null,
             Long.MIN_VALUE,
             0L,
@@ -102,7 +102,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "Log10UnsignedLongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             k -> null,
             BigInteger.ZERO,
             BigInteger.ZERO,
@@ -114,7 +114,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "Log10DoubleEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             k -> null,
             Double.NEGATIVE_INFINITY,
             0d,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
index e884b63c52591..a25fc66ab2d73 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
 
@@ -55,7 +54,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "LogConstantEvaluator[value=Attribute[channel=0]]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::log,
             Math.nextUp(0d),
             Math.nextDown(1d),
@@ -70,12 +69,12 @@ public static Iterable<Object[]> parameters() {
                 "value",
                 (b, l) -> Math.log10(l) / Math.log10(b),
                 List.of(
-                    new TestCaseSupplier.TypedDataSupplier("<gt0 double>", () -> Math.nextUp(0d), DataTypes.DOUBLE),
-                    new TestCaseSupplier.TypedDataSupplier("<lt1 double>", () -> Math.nextDown(1d), DataTypes.DOUBLE)
+                    new TestCaseSupplier.TypedDataSupplier("<gt0 double>", () -> Math.nextUp(0d), DataType.DOUBLE),
+                    new TestCaseSupplier.TypedDataSupplier("<lt1 double>", () -> Math.nextDown(1d), DataType.DOUBLE)
                 ),
                 List.of(
-                    new TestCaseSupplier.TypedDataSupplier("<gt0 double>", () -> Math.nextUp(0d), DataTypes.DOUBLE),
-                    new TestCaseSupplier.TypedDataSupplier("<lt1 double>", () -> Math.nextDown(1d), DataTypes.DOUBLE)
+                    new TestCaseSupplier.TypedDataSupplier("<gt0 double>", () -> Math.nextUp(0d), DataType.DOUBLE),
+                    new TestCaseSupplier.TypedDataSupplier("<lt1 double>", () -> Math.nextDown(1d), DataType.DOUBLE)
                 ),
                 List.of()
             )
@@ -143,12 +142,12 @@ public static Iterable<Object[]> parameters() {
                 "value",
                 (b, l) -> Math.log10(l) / Math.log10(b),
                 List.of(
-                    new TestCaseSupplier.TypedDataSupplier("<gt0 double>", () -> Math.nextUp(0d), DataTypes.DOUBLE),
-                    new TestCaseSupplier.TypedDataSupplier("<lt1 double>", () -> Math.nextDown(1d), DataTypes.DOUBLE)
+                    new TestCaseSupplier.TypedDataSupplier("<gt0 double>", () -> Math.nextUp(0d), DataType.DOUBLE),
+                    new TestCaseSupplier.TypedDataSupplier("<lt1 double>", () -> Math.nextDown(1d), DataType.DOUBLE)
                 ),
                 List.of(
-                    new TestCaseSupplier.TypedDataSupplier("<gt1 double>", () -> Math.nextUp(1d), DataTypes.DOUBLE),
-                    new TestCaseSupplier.TypedDataSupplier("<largest double>", () -> Double.MAX_VALUE, DataTypes.DOUBLE)
+                    new TestCaseSupplier.TypedDataSupplier("<gt1 double>", () -> Math.nextUp(1d), DataType.DOUBLE),
+                    new TestCaseSupplier.TypedDataSupplier("<largest double>", () -> Double.MAX_VALUE, DataType.DOUBLE)
                 ),
                 List.of()
             )
@@ -162,12 +161,12 @@ public static Iterable<Object[]> parameters() {
                 "value",
                 (b, l) -> Math.log10(l) / Math.log10(b),
                 List.of(
-                    new TestCaseSupplier.TypedDataSupplier("<gt0 double>", () -> Math.nextUp(1d), DataTypes.DOUBLE),
-                    new TestCaseSupplier.TypedDataSupplier("<lt1 double>", () -> Double.MAX_VALUE, DataTypes.DOUBLE)
+                    new TestCaseSupplier.TypedDataSupplier("<gt0 double>", () -> Math.nextUp(1d), DataType.DOUBLE),
+                    new TestCaseSupplier.TypedDataSupplier("<lt1 double>", () -> Double.MAX_VALUE, DataType.DOUBLE)
                 ),
                 List.of(
-                    new TestCaseSupplier.TypedDataSupplier("<gt1 double>", () -> Math.nextUp(0d), DataTypes.DOUBLE),
-                    new TestCaseSupplier.TypedDataSupplier("<largest double>", () -> Math.nextDown(1d), DataTypes.DOUBLE)
+                    new TestCaseSupplier.TypedDataSupplier("<gt1 double>", () -> Math.nextUp(0d), DataType.DOUBLE),
+                    new TestCaseSupplier.TypedDataSupplier("<largest double>", () -> Math.nextDown(1d), DataType.DOUBLE)
                 ),
                 List.of()
             )
@@ -197,7 +196,7 @@ public static Iterable<Object[]> parameters() {
 
     @Override
     protected DataType expectedType(List<DataType> argTypes) {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/NowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/NowTests.java
index ed5a80cb81b43..2c1322abf8cda 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/NowTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/NowTests.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now;
@@ -42,7 +42,7 @@ public static Iterable<Object[]> parameters() {
                     () -> new TestCaseSupplier.TestCase(
                         List.of(),
                         matchesPattern("LiteralsEvaluator\\[lit=.*\\]"),
-                        DataTypes.DATETIME,
+                        DataType.DATETIME,
                         equalTo(EsqlTestUtils.TEST_CFG.now().toInstant().toEpochMilli())
                     )
                 )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java
index bee3a495a5aa5..c21082b905962 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -33,9 +33,9 @@ public PiTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSup
     public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Pi Test", () -> {
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(1, DataTypes.INTEGER, "foo")),
+                List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")),
                 "LiteralsEvaluator[lit=3.141592653589793]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 equalTo(Math.PI)
             );
         })));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
index acc1a3a10ba8d..855e3070d442f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
 
@@ -50,8 +49,8 @@ public static Iterable<Object[]> parameters() {
                 // 143^143 is still representable, but 144^144 is infinite
                 TestCaseSupplier.castToDoubleSuppliersFromRange(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY),
                 List.of(
-                    new TestCaseSupplier.TypedDataSupplier("<0 double>", () -> 0d, DataTypes.DOUBLE),
-                    new TestCaseSupplier.TypedDataSupplier("<-0 double>", () -> -0d, DataTypes.DOUBLE)
+                    new TestCaseSupplier.TypedDataSupplier("<0 double>", () -> 0d, DataType.DOUBLE),
+                    new TestCaseSupplier.TypedDataSupplier("<-0 double>", () -> -0d, DataType.DOUBLE)
                 ),
                 List.of()
             )
@@ -83,7 +82,7 @@ public static Iterable<Object[]> parameters() {
 
     @Override
     protected DataType expectedType(List<DataType> argTypes) {
-        return DataTypes.DOUBLE;
+        return DataType.DOUBLE;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundTests.java
index 115ce6b7019c6..5e19d5f606034 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.math.Maths;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -26,7 +25,6 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.test.ESTestCase.randomDouble;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -42,7 +40,7 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(
             supplier(
                 "<double>",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 () -> 1 / randomDouble(),
                 "RoundDoubleNoDecimalsEvaluator[val=Attribute[channel=0]]",
                 d -> Maths.round(d, 0)
@@ -51,9 +49,9 @@ public static Iterable<Object[]> parameters() {
         suppliers.add(
             supplier(
                 "<double>, <integer>",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 () -> 1 / randomDouble(),
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 () -> between(-30, 30),
                 "RoundDoubleEvaluator[val=Attribute[channel=0], decimals=CastIntToLongEvaluator[v=Attribute[channel=1]]]",
                 Maths::round
@@ -67,29 +65,29 @@ public static Iterable<Object[]> parameters() {
             (nullPosition, nullData, original) -> original
         );
 
-        suppliers.add(new TestCaseSupplier("two doubles", List.of(DataTypes.DOUBLE, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("two doubles", List.of(DataType.DOUBLE, DataType.INTEGER), () -> {
             double number1 = 1 / randomDouble();
             double number2 = 1 / randomDouble();
             int precision = between(-30, 30);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(List.of(number1, number2), DataTypes.DOUBLE, "number"),
-                    new TestCaseSupplier.TypedData(precision, DataTypes.INTEGER, "decimals")
+                    new TestCaseSupplier.TypedData(List.of(number1, number2), DataType.DOUBLE, "number"),
+                    new TestCaseSupplier.TypedData(precision, DataType.INTEGER, "decimals")
                 ),
                 "RoundDoubleEvaluator[val=Attribute[channel=0], decimals=CastIntToLongEvaluator[v=Attribute[channel=1]]]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 is(nullValue())
             ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
                 .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value");
         }));
 
         // Integer or Long without a decimals parameter is a noop
-        suppliers.add(supplier("<integer>", DataTypes.INTEGER, ESTestCase::randomInt, "Attribute[channel=0]", Function.identity()));
-        suppliers.add(supplier("<long>", DataTypes.LONG, ESTestCase::randomLong, "Attribute[channel=0]", Function.identity()));
+        suppliers.add(supplier("<integer>", DataType.INTEGER, ESTestCase::randomInt, "Attribute[channel=0]", Function.identity()));
+        suppliers.add(supplier("<long>", DataType.LONG, ESTestCase::randomLong, "Attribute[channel=0]", Function.identity()));
         suppliers.add(
             supplier(
                 "<unsigned_long>",
-                DataTypes.UNSIGNED_LONG,
+                DataType.UNSIGNED_LONG,
                 ESTestCase::randomLong,
                 "Attribute[channel=0]",
                 NumericUtils::unsignedLongAsBigInteger
@@ -136,7 +134,7 @@ public static Iterable<Object[]> parameters() {
     private static TestCaseSupplier supplier(double v, double expected) {
         return supplier(
             "round(" + v + ") -> " + expected,
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             () -> v,
             "RoundDoubleNoDecimalsEvaluator[val=Attribute[channel=0]]",
             value -> expected
@@ -146,9 +144,9 @@ private static TestCaseSupplier supplier(double v, double expected) {
     private static TestCaseSupplier supplier(double v, int decimals, double expected) {
         return supplier(
             "round(" + v + ", " + decimals + ") -> " + expected,
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             () -> v,
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             () -> decimals,
             "RoundDoubleEvaluator[val=Attribute[channel=0], decimals=CastIntToLongEvaluator[v=Attribute[channel=1]]]",
             (value, de) -> expected
@@ -158,9 +156,9 @@ private static TestCaseSupplier supplier(double v, int decimals, double expected
     private static TestCaseSupplier supplier(long v, int decimals, long expected) {
         return supplier(
             "round(" + v + "L, " + decimals + ") -> " + expected,
-            DataTypes.LONG,
+            DataType.LONG,
             () -> v,
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             () -> decimals,
             "RoundLongEvaluator[val=Attribute[channel=0], decimals=CastIntToLongEvaluator[v=Attribute[channel=1]]]",
             (value, de) -> expected
@@ -170,9 +168,9 @@ private static TestCaseSupplier supplier(long v, int decimals, long expected) {
     private static TestCaseSupplier supplier(int v, int decimals, int expected) {
         return supplier(
             "round(" + v + ", " + decimals + ") -> " + expected,
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             () -> v,
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             () -> decimals,
             "RoundIntEvaluator[val=Attribute[channel=0], decimals=CastIntToLongEvaluator[v=Attribute[channel=1]]]",
             (value, de) -> expected
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java
index 8a6d88fb399a1..89c2d07c4470a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -34,7 +34,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "SignumIntEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             i -> (double) Math.signum(i),
             Integer.MIN_VALUE,
             Integer.MAX_VALUE,
@@ -44,7 +44,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             "SignumLongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             l -> (double) Math.signum(l),
             Long.MIN_VALUE,
             Long.MAX_VALUE,
@@ -54,7 +54,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "SignumUnsignedLongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             ul -> Math.signum(NumericUtils.unsignedLongToDouble(NumericUtils.asLongUnsigned(ul))),
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
@@ -63,7 +63,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "SignumDoubleEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::signum,
             -Double.MAX_VALUE,
             Double.MAX_VALUE,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java
index c7fbe713fab0c..a1d5b8523175c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -37,7 +37,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "SqrtIntEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::sqrt,
             0,
             Integer.MAX_VALUE,
@@ -46,7 +46,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             "SqrtLongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::sqrt,
             0,
             Long.MAX_VALUE,
@@ -55,7 +55,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryUnsignedLong(
             suppliers,
             "SqrtUnsignedLongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             ul -> Math.sqrt(ul == null ? null : unsignedLongToDouble(NumericUtils.asLongUnsigned(ul))),
             BigInteger.ZERO,
             UNSIGNED_LONG_MAX,
@@ -64,7 +64,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "SqrtDoubleEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             Math::sqrt,
             -0d,
             Double.MAX_VALUE,
@@ -76,7 +76,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "SqrtIntEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             k -> null,
             Integer.MIN_VALUE,
             -1,
@@ -88,7 +88,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             "SqrtLongEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             k -> null,
             Long.MIN_VALUE,
             -1,
@@ -100,7 +100,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "SqrtDoubleEvaluator[val=" + read + "]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             k -> null,
             Double.NEGATIVE_INFINITY,
             -Double.MIN_VALUE,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java
index 57448df4ec788..aa64dfc6af90d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -33,9 +33,9 @@ public TauTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSu
     public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Tau Test", () -> {
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(1, DataTypes.INTEGER, "foo")),
+                List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")),
                 "LiteralsEvaluator[lit=6.283185307179586]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 equalTo(Tau.TAU)
             );
         })));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
index ce7cd68198753..2ea79d8a165c6 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
@@ -50,7 +49,7 @@ protected static void booleans(
         String evaluatorName,
         BiFunction<Integer, Stream<Boolean>, Matcher<Object>> matcher
     ) {
-        booleans(cases, name, evaluatorName, DataTypes.BOOLEAN, matcher);
+        booleans(cases, name, evaluatorName, DataType.BOOLEAN, matcher);
     }
 
     /**
@@ -66,9 +65,9 @@ protected static void booleans(
         cases.add(
             new TestCaseSupplier(
                 name + "(false)",
-                List.of(DataTypes.BOOLEAN),
+                List.of(DataType.BOOLEAN),
                 () -> new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(List.of(false), DataTypes.BOOLEAN, "field")),
+                    List.of(new TestCaseSupplier.TypedData(List.of(false), DataType.BOOLEAN, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(1, Stream.of(false))
@@ -78,9 +77,9 @@ protected static void booleans(
         cases.add(
             new TestCaseSupplier(
                 name + "(true)",
-                List.of(DataTypes.BOOLEAN),
+                List.of(DataType.BOOLEAN),
                 () -> new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(List.of(true), DataTypes.BOOLEAN, "field")),
+                    List.of(new TestCaseSupplier.TypedData(List.of(true), DataType.BOOLEAN, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(1, Stream.of(true))
@@ -88,11 +87,11 @@ protected static void booleans(
             )
         );
         for (Block.MvOrdering ordering : Block.MvOrdering.values()) {
-            cases.add(new TestCaseSupplier(name + "(<booleans>) " + ordering, List.of(DataTypes.BOOLEAN), () -> {
+            cases.add(new TestCaseSupplier(name + "(<booleans>) " + ordering, List.of(DataType.BOOLEAN), () -> {
                 List<Boolean> mvData = randomList(2, 100, ESTestCase::randomBoolean);
                 putInOrder(mvData, ordering);
                 return new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(mvData, DataTypes.BOOLEAN, "field")),
+                    List.of(new TestCaseSupplier.TypedData(mvData, DataType.BOOLEAN, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(mvData.size(), mvData.stream())
@@ -123,8 +122,8 @@ protected static void bytesRefs(
         Function<DataType, DataType> expectedDataType,
         BiFunction<Integer, Stream<BytesRef>, Matcher<Object>> matcher
     ) {
-        for (DataType type : new DataType[] { DataTypes.KEYWORD, DataTypes.TEXT, DataTypes.IP, DataTypes.VERSION }) {
-            if (type != DataTypes.IP) {
+        for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.IP, DataType.VERSION }) {
+            if (type != DataType.IP) {
                 cases.add(
                     new TestCaseSupplier(
                         name + "(empty " + type.typeName() + ")",
@@ -171,7 +170,7 @@ protected static void doubles(
         String evaluatorName,
         BiFunction<Integer, DoubleStream, Matcher<Object>> matcher
     ) {
-        doubles(cases, name, evaluatorName, DataTypes.DOUBLE, matcher);
+        doubles(cases, name, evaluatorName, DataType.DOUBLE, matcher);
     }
 
     /**
@@ -187,30 +186,30 @@ protected static void doubles(
         cases.add(
             new TestCaseSupplier(
                 name + "(0.0)",
-                List.of(DataTypes.DOUBLE),
+                List.of(DataType.DOUBLE),
                 () -> new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(List.of(0.0), DataTypes.DOUBLE, "field")),
+                    List.of(new TestCaseSupplier.TypedData(List.of(0.0), DataType.DOUBLE, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(1, DoubleStream.of(0.0))
                 )
             )
         );
-        cases.add(new TestCaseSupplier(name + "(double)", List.of(DataTypes.DOUBLE), () -> {
+        cases.add(new TestCaseSupplier(name + "(double)", List.of(DataType.DOUBLE), () -> {
             double mvData = randomDouble();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(List.of(mvData), DataTypes.DOUBLE, "field")),
+                List.of(new TestCaseSupplier.TypedData(List.of(mvData), DataType.DOUBLE, "field")),
                 evaluatorName + "[field=Attribute[channel=0]]",
                 expectedDataType,
                 matcher.apply(1, DoubleStream.of(mvData))
             );
         }));
         for (Block.MvOrdering ordering : Block.MvOrdering.values()) {
-            cases.add(new TestCaseSupplier(name + "(<double>) " + ordering, List.of(DataTypes.DOUBLE), () -> {
+            cases.add(new TestCaseSupplier(name + "(<double>) " + ordering, List.of(DataType.DOUBLE), () -> {
                 List<Double> mvData = randomList(1, 100, ESTestCase::randomDouble);
                 putInOrder(mvData, ordering);
                 return new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(mvData, DataTypes.DOUBLE, "field")),
+                    List.of(new TestCaseSupplier.TypedData(mvData, DataType.DOUBLE, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(mvData.size(), mvData.stream().mapToDouble(Double::doubleValue))
@@ -228,7 +227,7 @@ protected static void ints(
         String evaluatorName,
         BiFunction<Integer, IntStream, Matcher<Object>> matcher
     ) {
-        ints(cases, name, evaluatorName, DataTypes.INTEGER, matcher);
+        ints(cases, name, evaluatorName, DataType.INTEGER, matcher);
     }
 
     /**
@@ -244,30 +243,30 @@ protected static void ints(
         cases.add(
             new TestCaseSupplier(
                 name + "(0)",
-                List.of(DataTypes.INTEGER),
+                List.of(DataType.INTEGER),
                 () -> new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(List.of(0), DataTypes.INTEGER, "field")),
+                    List.of(new TestCaseSupplier.TypedData(List.of(0), DataType.INTEGER, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(1, IntStream.of(0))
                 )
             )
         );
-        cases.add(new TestCaseSupplier(name + "(int)", List.of(DataTypes.INTEGER), () -> {
+        cases.add(new TestCaseSupplier(name + "(int)", List.of(DataType.INTEGER), () -> {
             int data = randomInt();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(List.of(data), DataTypes.INTEGER, "field")),
+                List.of(new TestCaseSupplier.TypedData(List.of(data), DataType.INTEGER, "field")),
                 evaluatorName + "[field=Attribute[channel=0]]",
                 expectedDataType,
                 matcher.apply(1, IntStream.of(data))
             );
         }));
         for (Block.MvOrdering ordering : Block.MvOrdering.values()) {
-            cases.add(new TestCaseSupplier(name + "(<ints>) " + ordering, List.of(DataTypes.INTEGER), () -> {
+            cases.add(new TestCaseSupplier(name + "(<ints>) " + ordering, List.of(DataType.INTEGER), () -> {
                 List<Integer> mvData = randomList(1, 100, ESTestCase::randomInt);
                 putInOrder(mvData, ordering);
                 return new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(mvData, DataTypes.INTEGER, "field")),
+                    List.of(new TestCaseSupplier.TypedData(mvData, DataType.INTEGER, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(mvData.size(), mvData.stream().mapToInt(Integer::intValue))
@@ -285,7 +284,7 @@ protected static void longs(
         String evaluatorName,
         BiFunction<Integer, LongStream, Matcher<Object>> matcher
     ) {
-        longs(cases, name, evaluatorName, DataTypes.LONG, matcher);
+        longs(cases, name, evaluatorName, DataType.LONG, matcher);
     }
 
     /**
@@ -301,30 +300,30 @@ protected static void longs(
         cases.add(
             new TestCaseSupplier(
                 name + "(0L)",
-                List.of(DataTypes.LONG),
+                List.of(DataType.LONG),
                 () -> new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(List.of(0L), DataTypes.LONG, "field")),
+                    List.of(new TestCaseSupplier.TypedData(List.of(0L), DataType.LONG, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(1, LongStream.of(0L))
                 )
             )
         );
-        cases.add(new TestCaseSupplier(name + "(long)", List.of(DataTypes.LONG), () -> {
+        cases.add(new TestCaseSupplier(name + "(long)", List.of(DataType.LONG), () -> {
             long data = randomLong();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(List.of(data), DataTypes.LONG, "field")),
+                List.of(new TestCaseSupplier.TypedData(List.of(data), DataType.LONG, "field")),
                 evaluatorName + "[field=Attribute[channel=0]]",
                 expectedDataType,
                 matcher.apply(1, LongStream.of(data))
             );
         }));
         for (Block.MvOrdering ordering : Block.MvOrdering.values()) {
-            cases.add(new TestCaseSupplier(name + "(<longs>) " + ordering, List.of(DataTypes.LONG), () -> {
+            cases.add(new TestCaseSupplier(name + "(<longs>) " + ordering, List.of(DataType.LONG), () -> {
                 List<Long> mvData = randomList(1, 100, ESTestCase::randomLong);
                 putInOrder(mvData, ordering);
                 return new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(mvData, DataTypes.LONG, "field")),
+                    List.of(new TestCaseSupplier.TypedData(mvData, DataType.LONG, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(mvData.size(), mvData.stream().mapToLong(Long::longValue))
@@ -342,7 +341,7 @@ protected static void dateTimes(
         String evaluatorName,
         BiFunction<Integer, LongStream, Matcher<Object>> matcher
     ) {
-        dateTimes(cases, name, evaluatorName, DataTypes.DATETIME, matcher);
+        dateTimes(cases, name, evaluatorName, DataType.DATETIME, matcher);
     }
 
     /**
@@ -358,30 +357,30 @@ protected static void dateTimes(
         cases.add(
             new TestCaseSupplier(
                 name + "(epoch)",
-                List.of(DataTypes.DATETIME),
+                List.of(DataType.DATETIME),
                 () -> new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(List.of(0L), DataTypes.DATETIME, "field")),
+                    List.of(new TestCaseSupplier.TypedData(List.of(0L), DataType.DATETIME, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(1, LongStream.of(0L))
                 )
             )
         );
-        cases.add(new TestCaseSupplier(name + "(date)", List.of(DataTypes.DATETIME), () -> {
+        cases.add(new TestCaseSupplier(name + "(date)", List.of(DataType.DATETIME), () -> {
             long data = randomLong();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(List.of(data), DataTypes.DATETIME, "field")),
+                List.of(new TestCaseSupplier.TypedData(List.of(data), DataType.DATETIME, "field")),
                 evaluatorName + "[field=Attribute[channel=0]]",
                 expectedDataType,
                 matcher.apply(1, LongStream.of(data))
             );
         }));
         for (Block.MvOrdering ordering : Block.MvOrdering.values()) {
-            cases.add(new TestCaseSupplier(name + "(<dates>) " + ordering, List.of(DataTypes.DATETIME), () -> {
+            cases.add(new TestCaseSupplier(name + "(<dates>) " + ordering, List.of(DataType.DATETIME), () -> {
                 List<Long> mvData = randomList(1, 100, ESTestCase::randomLong);
                 putInOrder(mvData, ordering);
                 return new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(mvData, DataTypes.DATETIME, "field")),
+                    List.of(new TestCaseSupplier.TypedData(mvData, DataType.DATETIME, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(mvData.size(), mvData.stream().mapToLong(Long::longValue))
@@ -400,7 +399,7 @@ protected static void geoPoints(
         String evaluatorName,
         BiFunction<Integer, Stream<BytesRef>, Matcher<Object>> matcher
     ) {
-        geoPoints(cases, name, evaluatorName, DataTypes.GEO_POINT, matcher);
+        geoPoints(cases, name, evaluatorName, DataType.GEO_POINT, matcher);
     }
 
     /**
@@ -415,7 +414,7 @@ protected static void geoPoints(
         DataType expectedDataType,
         BiFunction<Integer, Stream<BytesRef>, Matcher<Object>> matcher
     ) {
-        spatial(cases, name, evaluatorName, DataTypes.GEO_POINT, expectedDataType, GEO, GeometryTestUtils::randomPoint, matcher);
+        spatial(cases, name, evaluatorName, DataType.GEO_POINT, expectedDataType, GEO, GeometryTestUtils::randomPoint, matcher);
     }
 
     /**
@@ -428,7 +427,7 @@ protected static void cartesianPoints(
         String evaluatorName,
         BiFunction<Integer, Stream<BytesRef>, Matcher<Object>> matcher
     ) {
-        cartesianPoints(cases, name, evaluatorName, DataTypes.CARTESIAN_POINT, matcher);
+        cartesianPoints(cases, name, evaluatorName, DataType.CARTESIAN_POINT, matcher);
     }
 
     /**
@@ -443,7 +442,7 @@ protected static void cartesianPoints(
         DataType expectedDataType,
         BiFunction<Integer, Stream<BytesRef>, Matcher<Object>> matcher
     ) {
-        spatial(cases, name, evaluatorName, DataTypes.CARTESIAN_POINT, expectedDataType, CARTESIAN, ShapeTestUtils::randomPoint, matcher);
+        spatial(cases, name, evaluatorName, DataType.CARTESIAN_POINT, expectedDataType, CARTESIAN, ShapeTestUtils::randomPoint, matcher);
     }
 
     /**
@@ -462,7 +461,7 @@ protected static void geoShape(
             cases,
             name,
             evaluatorName,
-            DataTypes.GEO_SHAPE,
+            DataType.GEO_SHAPE,
             expectedDataType,
             GEO,
             () -> rarely() ? GeometryTestUtils.randomGeometry(randomBoolean()) : GeometryTestUtils.randomPoint(),
@@ -486,7 +485,7 @@ protected static void cartesianShape(
             cases,
             name,
             evaluatorName,
-            DataTypes.CARTESIAN_SHAPE,
+            DataType.CARTESIAN_SHAPE,
             expectedDataType,
             CARTESIAN,
             () -> rarely() ? ShapeTestUtils.randomGeometry(randomBoolean()) : ShapeTestUtils.randomPoint(),
@@ -539,7 +538,7 @@ protected static void unsignedLongs(
         String evaluatorName,
         BiFunction<Integer, Stream<BigInteger>, Matcher<Object>> matcher
     ) {
-        unsignedLongs(cases, name, evaluatorName, DataTypes.UNSIGNED_LONG, matcher);
+        unsignedLongs(cases, name, evaluatorName, DataType.UNSIGNED_LONG, matcher);
     }
 
     /**
@@ -555,12 +554,12 @@ protected static void unsignedLongs(
         cases.add(
             new TestCaseSupplier(
                 name + "(0UL)",
-                List.of(DataTypes.UNSIGNED_LONG),
+                List.of(DataType.UNSIGNED_LONG),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
                         new TestCaseSupplier.TypedData(
                             List.of(NumericUtils.asLongUnsigned(BigInteger.ZERO)),
-                            DataTypes.UNSIGNED_LONG,
+                            DataType.UNSIGNED_LONG,
                             "field"
                         )
                     ),
@@ -570,21 +569,21 @@ protected static void unsignedLongs(
                 )
             )
         );
-        cases.add(new TestCaseSupplier(name + "(unsigned long)", List.of(DataTypes.UNSIGNED_LONG), () -> {
+        cases.add(new TestCaseSupplier(name + "(unsigned long)", List.of(DataType.UNSIGNED_LONG), () -> {
             long data = randomLong();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(List.of(data), DataTypes.UNSIGNED_LONG, "field")),
+                List.of(new TestCaseSupplier.TypedData(List.of(data), DataType.UNSIGNED_LONG, "field")),
                 evaluatorName + "[field=Attribute[channel=0]]",
                 expectedDataType,
                 matcher.apply(1, Stream.of(NumericUtils.unsignedLongAsBigInteger(data)))
             );
         }));
         for (Block.MvOrdering ordering : Block.MvOrdering.values()) {
-            cases.add(new TestCaseSupplier(name + "(<unsigned longs>) " + ordering, List.of(DataTypes.UNSIGNED_LONG), () -> {
+            cases.add(new TestCaseSupplier(name + "(<unsigned longs>) " + ordering, List.of(DataType.UNSIGNED_LONG), () -> {
                 List<Long> mvData = randomList(1, 100, ESTestCase::randomLong);
                 putInOrder(mvData, ordering);
                 return new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(mvData, DataTypes.UNSIGNED_LONG, "field")),
+                    List.of(new TestCaseSupplier.TypedData(mvData, DataType.UNSIGNED_LONG, "field")),
                     evaluatorName + "[field=Attribute[channel=0]]",
                     expectedDataType,
                     matcher.apply(mvData.size(), mvData.stream().map(NumericUtils::unsignedLongAsBigInteger))
@@ -621,7 +620,7 @@ private static <T extends Comparable<T>> void putInOrder(List<T> mvData, Block.M
 
     protected final DataType[] representableNumerics() {
         // TODO numeric should only include representable numbers but that is a change for a followup
-        return DataTypes.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new);
+        return DataType.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new);
     }
 
     protected DataType expectedType(List<DataType> argTypes) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java
index 0ad43f520daf3..966a5a590e256 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -41,14 +40,14 @@ public static Iterable<Object[]> parameters() {
             return equalTo(sum.value() / size);
         };
         List<TestCaseSupplier> cases = new ArrayList<>();
-        doubles(cases, "mv_avg", "MvAvg", DataTypes.DOUBLE, avg);
-        ints(cases, "mv_avg", "MvAvg", DataTypes.DOUBLE, (size, data) -> avg.apply(size, data.mapToDouble(v -> (double) v)));
-        longs(cases, "mv_avg", "MvAvg", DataTypes.DOUBLE, (size, data) -> avg.apply(size, data.mapToDouble(v -> (double) v)));
+        doubles(cases, "mv_avg", "MvAvg", DataType.DOUBLE, avg);
+        ints(cases, "mv_avg", "MvAvg", DataType.DOUBLE, (size, data) -> avg.apply(size, data.mapToDouble(v -> (double) v)));
+        longs(cases, "mv_avg", "MvAvg", DataType.DOUBLE, (size, data) -> avg.apply(size, data.mapToDouble(v -> (double) v)));
         unsignedLongs(
             cases,
             "mv_avg",
             "MvAvg",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             /*
              * Converting strait from BigInteger to double will round differently.
              * So we have to go back to encoded `long` and then convert to double
@@ -71,6 +70,6 @@ protected DataType[] supportedTypes() {
 
     @Override
     protected DataType expectedType(List<DataType> argTypes) {
-        return DataTypes.DOUBLE;  // Averages are always a double
+        return DataType.DOUBLE;  // Averages are always a double
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java
index 3b4ab2b5cfaa0..39ef5eefe9287 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -33,11 +32,11 @@ public MvConcatTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testC
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType fieldType : DataTypes.types()) {
+        for (DataType fieldType : DataType.types()) {
             if (EsqlDataTypes.isString(fieldType) == false) {
                 continue;
             }
-            for (DataType delimType : DataTypes.types()) {
+            for (DataType delimType : DataType.types()) {
                 if (EsqlDataTypes.isString(delimType) == false) {
                     continue;
                 }
@@ -62,7 +61,7 @@ public static Iterable<Object[]> parameters() {
                                 new TestCaseSupplier.TypedData(new BytesRef(delim), delimType, "delim")
                             ),
                             "MvConcat[field=Attribute[channel=0], delim=Attribute[channel=1]]",
-                            DataTypes.KEYWORD,
+                            DataType.KEYWORD,
                             equalTo(new BytesRef(expected))
                         );
                     }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java
index 9c379990e4707..8733dc0d25c40 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
 import java.util.ArrayList;
@@ -30,17 +29,17 @@ public MvCountTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCa
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> cases = new ArrayList<>();
-        booleans(cases, "mv_count", "MvCount", DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        bytesRefs(cases, "mv_count", "MvCount", t -> DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        doubles(cases, "mv_count", "MvCount", DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        ints(cases, "mv_count", "MvCount", DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        longs(cases, "mv_count", "MvCount", DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        unsignedLongs(cases, "mv_count", "MvCount", DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        dateTimes(cases, "mv_count", "MvCount", DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        geoPoints(cases, "mv_count", "MvCount", DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        cartesianPoints(cases, "mv_count", "MvCount", DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        geoShape(cases, "mv_count", "MvCount", DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
-        cartesianShape(cases, "mv_count", "MvCount", DataTypes.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        booleans(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        bytesRefs(cases, "mv_count", "MvCount", t -> DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        doubles(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        ints(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        longs(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        unsignedLongs(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        dateTimes(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        geoPoints(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        cartesianPoints(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        geoShape(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
+        cartesianShape(cases, "mv_count", "MvCount", DataType.INTEGER, (size, values) -> equalTo(Math.toIntExact(values.count())));
         return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, cases)));
     }
 
@@ -56,6 +55,6 @@ protected DataType[] supportedTypes() {
 
     @Override
     protected DataType expectedType(List<DataType> argTypes) {
-        return DataTypes.INTEGER;
+        return DataType.INTEGER;
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
index 5f275e4c52d43..f202a8033ffc9 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
@@ -44,9 +43,9 @@ public static Iterable<Object[]> parameters() {
         ints(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values.mapToObj(Integer::valueOf)));
         longs(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values.mapToObj(Long::valueOf)));
         cartesianPoints(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values));
-        cartesianShape(cases, "mv_dedupe", "MvDedupe", DataTypes.CARTESIAN_SHAPE, (size, values) -> getMatcher(values));
+        cartesianShape(cases, "mv_dedupe", "MvDedupe", DataType.CARTESIAN_SHAPE, (size, values) -> getMatcher(values));
         geoPoints(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values));
-        geoShape(cases, "mv_dedupe", "MvDedupe", DataTypes.GEO_SHAPE, (size, values) -> getMatcher(values));
+        geoShape(cases, "mv_dedupe", "MvDedupe", DataType.GEO_SHAPE, (size, values) -> getMatcher(values));
 
         // TODO switch extraction to BigInteger so this just works.
         // unsignedLongs(cases, "mv_dedupe", "MvDedupe", (size, values) -> getMatcher(values));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java
index abd9a39f944d4..1c24b1a8aae64 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
 import java.util.ArrayList;
@@ -31,17 +30,17 @@ public MvFirstTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCa
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> cases = new ArrayList<>();
-        booleans(cases, "mv_first", "MvFirst", DataTypes.BOOLEAN, (size, values) -> equalTo(values.findFirst().get()));
+        booleans(cases, "mv_first", "MvFirst", DataType.BOOLEAN, (size, values) -> equalTo(values.findFirst().get()));
         bytesRefs(cases, "mv_first", "MvFirst", Function.identity(), (size, values) -> equalTo(values.findFirst().get()));
-        doubles(cases, "mv_first", "MvFirst", DataTypes.DOUBLE, (size, values) -> equalTo(values.findFirst().getAsDouble()));
-        ints(cases, "mv_first", "MvFirst", DataTypes.INTEGER, (size, values) -> equalTo(values.findFirst().getAsInt()));
-        longs(cases, "mv_first", "MvFirst", DataTypes.LONG, (size, values) -> equalTo(values.findFirst().getAsLong()));
-        unsignedLongs(cases, "mv_first", "MvFirst", DataTypes.UNSIGNED_LONG, (size, values) -> equalTo(values.findFirst().get()));
-        dateTimes(cases, "mv_first", "MvFirst", DataTypes.DATETIME, (size, values) -> equalTo(values.findFirst().getAsLong()));
-        geoPoints(cases, "mv_first", "MvFirst", DataTypes.GEO_POINT, (size, values) -> equalTo(values.findFirst().get()));
-        cartesianPoints(cases, "mv_first", "MvFirst", DataTypes.CARTESIAN_POINT, (size, values) -> equalTo(values.findFirst().get()));
-        geoShape(cases, "mv_first", "MvFirst", DataTypes.GEO_SHAPE, (size, values) -> equalTo(values.findFirst().get()));
-        cartesianShape(cases, "mv_first", "MvFirst", DataTypes.CARTESIAN_SHAPE, (size, values) -> equalTo(values.findFirst().get()));
+        doubles(cases, "mv_first", "MvFirst", DataType.DOUBLE, (size, values) -> equalTo(values.findFirst().getAsDouble()));
+        ints(cases, "mv_first", "MvFirst", DataType.INTEGER, (size, values) -> equalTo(values.findFirst().getAsInt()));
+        longs(cases, "mv_first", "MvFirst", DataType.LONG, (size, values) -> equalTo(values.findFirst().getAsLong()));
+        unsignedLongs(cases, "mv_first", "MvFirst", DataType.UNSIGNED_LONG, (size, values) -> equalTo(values.findFirst().get()));
+        dateTimes(cases, "mv_first", "MvFirst", DataType.DATETIME, (size, values) -> equalTo(values.findFirst().getAsLong()));
+        geoPoints(cases, "mv_first", "MvFirst", DataType.GEO_POINT, (size, values) -> equalTo(values.findFirst().get()));
+        cartesianPoints(cases, "mv_first", "MvFirst", DataType.CARTESIAN_POINT, (size, values) -> equalTo(values.findFirst().get()));
+        geoShape(cases, "mv_first", "MvFirst", DataType.GEO_SHAPE, (size, values) -> equalTo(values.findFirst().get()));
+        cartesianShape(cases, "mv_first", "MvFirst", DataType.CARTESIAN_SHAPE, (size, values) -> equalTo(values.findFirst().get()));
         return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(false, cases)));
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java
index 286b372dce584..1b6fb482ea3d0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
 import java.util.ArrayList;
@@ -31,17 +30,17 @@ public MvLastTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCas
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> cases = new ArrayList<>();
-        booleans(cases, "mv_last", "MvLast", DataTypes.BOOLEAN, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
+        booleans(cases, "mv_last", "MvLast", DataType.BOOLEAN, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
         bytesRefs(cases, "mv_last", "MvLast", Function.identity(), (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
-        doubles(cases, "mv_last", "MvLast", DataTypes.DOUBLE, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsDouble()));
-        ints(cases, "mv_last", "MvLast", DataTypes.INTEGER, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsInt()));
-        longs(cases, "mv_last", "MvLast", DataTypes.LONG, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsLong()));
-        unsignedLongs(cases, "mv_last", "MvLast", DataTypes.UNSIGNED_LONG, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
-        dateTimes(cases, "mv_last", "MvLast", DataTypes.DATETIME, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsLong()));
-        geoPoints(cases, "mv_last", "MvLast", DataTypes.GEO_POINT, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
-        cartesianPoints(cases, "mv_last", "MvLast", DataTypes.CARTESIAN_POINT, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
-        geoShape(cases, "mv_last", "MvLast", DataTypes.GEO_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
-        cartesianShape(cases, "mv_last", "MvLast", DataTypes.CARTESIAN_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
+        doubles(cases, "mv_last", "MvLast", DataType.DOUBLE, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsDouble()));
+        ints(cases, "mv_last", "MvLast", DataType.INTEGER, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsInt()));
+        longs(cases, "mv_last", "MvLast", DataType.LONG, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsLong()));
+        unsignedLongs(cases, "mv_last", "MvLast", DataType.UNSIGNED_LONG, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
+        dateTimes(cases, "mv_last", "MvLast", DataType.DATETIME, (size, values) -> equalTo(values.reduce((f, s) -> s).getAsLong()));
+        geoPoints(cases, "mv_last", "MvLast", DataType.GEO_POINT, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
+        cartesianPoints(cases, "mv_last", "MvLast", DataType.CARTESIAN_POINT, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
+        geoShape(cases, "mv_last", "MvLast", DataType.GEO_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
+        cartesianShape(cases, "mv_last", "MvLast", DataType.CARTESIAN_SHAPE, (size, values) -> equalTo(values.reduce((f, s) -> s).get()));
         return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(false, cases)));
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java
index f6395074dbb80..4c324c916f861 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
 import java.math.BigInteger;
@@ -72,11 +71,11 @@ public static Iterable<Object[]> parameters() {
         cases.add(
             new TestCaseSupplier(
                 "mv_median(<1, 2>)",
-                List.of(DataTypes.INTEGER),
+                List.of(DataType.INTEGER),
                 () -> new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(List.of(1, 2), DataTypes.INTEGER, "field")),
+                    List.of(new TestCaseSupplier.TypedData(List.of(1, 2), DataType.INTEGER, "field")),
                     "MvMedian[field=Attribute[channel=0]]",
-                    DataTypes.INTEGER,
+                    DataType.INTEGER,
                     equalTo(1)
                 )
             )
@@ -84,11 +83,11 @@ public static Iterable<Object[]> parameters() {
         cases.add(
             new TestCaseSupplier(
                 "mv_median(<-1, -2>)",
-                List.of(DataTypes.INTEGER),
+                List.of(DataType.INTEGER),
                 () -> new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(List.of(-1, -2), DataTypes.INTEGER, "field")),
+                    List.of(new TestCaseSupplier.TypedData(List.of(-1, -2), DataType.INTEGER, "field")),
                     "MvMedian[field=Attribute[channel=0]]",
-                    DataTypes.INTEGER,
+                    DataType.INTEGER,
                     equalTo(-2)
                 )
             )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
index eba1bb623bb98..3ab17b78ff8e7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.geo.ShapeTestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -44,8 +44,8 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(
             anyNullIsNull(
                 suppliers,
-                (nullPosition, nullValueDataType, original) -> nullPosition == 0 && nullValueDataType == DataTypes.NULL
-                    ? DataTypes.NULL
+                (nullPosition, nullValueDataType, original) -> nullPosition == 0 && nullValueDataType == DataType.NULL
+                    ? DataType.NULL
                     : original.expectedType(),
                 (nullPosition, nullData, original) -> original
             )
@@ -59,282 +59,282 @@ protected Expression build(Source source, List<Expression> args) {
 
     private static void booleans(List<TestCaseSupplier> suppliers) {
         // Positive
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.BOOLEAN, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.BOOLEAN, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Boolean> field = randomList(1, 10, () -> randomBoolean());
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.BOOLEAN, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.BOOLEAN, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBooleanEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
         // Positive Start IndexOutofBound
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.BOOLEAN, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.BOOLEAN, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Boolean> field = randomList(1, 10, () -> randomBoolean());
             int length = field.size();
             int start = randomIntBetween(length, length + 1);
             int end = randomIntBetween(start, length + 10);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.BOOLEAN, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.BOOLEAN, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBooleanEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 nullValue()
             );
         }));
         // Positive End IndexOutofBound
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.BOOLEAN, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.BOOLEAN, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Boolean> field = randomList(1, 10, () -> randomBoolean());
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(length, length + 10);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.BOOLEAN, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.BOOLEAN, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBooleanEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(start == length - 1 ? field.get(start) : field.subList(start, length))
             );
         }));
         // Negative
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.BOOLEAN, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.BOOLEAN, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Boolean> field = randomList(1, 10, () -> randomBoolean());
             int length = field.size();
             int start = randomIntBetween(0 - length, -1);
             int end = randomIntBetween(start, -1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.BOOLEAN, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.BOOLEAN, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBooleanEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(start == end ? field.get(start + length) : field.subList(start + length, end + 1 + length))
             );
         }));
     }
 
     private static void ints(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.INTEGER, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Integer> field = randomList(1, 10, () -> randomInt());
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.INTEGER, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.INTEGER, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceIntEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
     }
 
     private static void longs(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.LONG, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.LONG, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Long> field = randomList(1, 10, () -> randomLong());
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.LONG, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.LONG, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceLongEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.LONG,
+                DataType.LONG,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.DATETIME, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.DATETIME, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Long> field = randomList(1, 10, () -> randomLong());
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.DATETIME, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.DATETIME, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceLongEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
     }
 
     private static void doubles(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.DOUBLE, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.DOUBLE, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Double> field = randomList(1, 10, () -> randomDouble());
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.DOUBLE, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.DOUBLE, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceDoubleEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
     }
 
     private static void bytesRefs(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.KEYWORD, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataTypes.KEYWORD).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD, DataType.INTEGER, DataType.INTEGER), () -> {
+            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.KEYWORD).value());
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.KEYWORD, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.KEYWORD, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.TEXT, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataTypes.TEXT).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT, DataType.INTEGER, DataType.INTEGER), () -> {
+            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value());
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.TEXT, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.TEXT, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.TEXT,
+                DataType.TEXT,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataTypes.IP).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.INTEGER, DataType.INTEGER), () -> {
+            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.IP).value());
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.IP, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.IP, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.IP,
+                DataType.IP,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.VERSION, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataTypes.VERSION).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.VERSION, DataType.INTEGER, DataType.INTEGER), () -> {
+            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.VERSION).value());
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.VERSION, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.VERSION, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.VERSION,
+                DataType.VERSION,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.GEO_POINT, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.GEO_POINT, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Object> field = randomList(1, 5, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint())));
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.GEO_POINT, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.GEO_POINT, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.GEO_POINT,
+                DataType.GEO_POINT,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.CARTESIAN_POINT, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.CARTESIAN_POINT, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Object> field = randomList(1, 5, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint())));
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.CARTESIAN_POINT, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.CARTESIAN_POINT, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.CARTESIAN_POINT,
+                DataType.CARTESIAN_POINT,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.GEO_SHAPE, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.GEO_SHAPE, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Object> field = randomList(1, 5, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))));
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.GEO_SHAPE, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.GEO_SHAPE, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.GEO_SHAPE,
+                DataType.GEO_SHAPE,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.CARTESIAN_SHAPE, DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.CARTESIAN_SHAPE, DataType.INTEGER, DataType.INTEGER), () -> {
             List<Object> field = randomList(1, 5, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))));
             int length = field.size();
             int start = randomIntBetween(0, length - 1);
             int end = randomIntBetween(start, length - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.CARTESIAN_SHAPE, "field"),
-                    new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                    new TestCaseSupplier.TypedData(end, DataTypes.INTEGER, "end")
+                    new TestCaseSupplier.TypedData(field, DataType.CARTESIAN_SHAPE, "field"),
+                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                    new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end")
                 ),
                 "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]",
-                DataTypes.CARTESIAN_SHAPE,
+                DataType.CARTESIAN_SHAPE,
                 equalTo(start == end ? field.get(start) : field.subList(start, end + 1))
             );
         }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
index 9019de87256f9..7c6413e590bfe 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -47,16 +47,16 @@ protected Expression build(Source source, List<Expression> args) {
     }
 
     private static void booleans(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.BOOLEAN, DataTypes.KEYWORD), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.BOOLEAN, DataType.KEYWORD), () -> {
             List<Boolean> field = randomList(1, 10, () -> randomBoolean());
             BytesRef order = new BytesRef("ASC");
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.BOOLEAN, "field"),
-                    new TestCaseSupplier.TypedData(order, DataTypes.KEYWORD, "order").forceLiteral()
+                    new TestCaseSupplier.TypedData(field, DataType.BOOLEAN, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
                 "MvSort" + ElementType.BOOLEAN + "[field=Attribute[channel=0], order=true]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
             );
         }));
@@ -64,120 +64,120 @@ private static void booleans(List<TestCaseSupplier> suppliers) {
     }
 
     private static void ints(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.KEYWORD), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.INTEGER, DataType.KEYWORD), () -> {
             List<Integer> field = randomList(1, 10, () -> randomInt());
             BytesRef order = new BytesRef("DESC");
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.INTEGER, "field"),
-                    new TestCaseSupplier.TypedData(order, DataTypes.KEYWORD, "order").forceLiteral()
+                    new TestCaseSupplier.TypedData(field, DataType.INTEGER, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
                 "MvSort" + ElementType.INT + "[field=Attribute[channel=0], order=false]",
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
             );
         }));
     }
 
     private static void longs(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.LONG, DataTypes.KEYWORD), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.LONG, DataType.KEYWORD), () -> {
             List<Long> field = randomList(1, 10, () -> randomLong());
             BytesRef order = new BytesRef("ASC");
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.LONG, "field"),
-                    new TestCaseSupplier.TypedData(order, DataTypes.KEYWORD, "order").forceLiteral()
+                    new TestCaseSupplier.TypedData(field, DataType.LONG, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
                 "MvSort" + ElementType.LONG + "[field=Attribute[channel=0], order=true]",
-                DataTypes.LONG,
+                DataType.LONG,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.DATETIME, DataTypes.KEYWORD), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.DATETIME, DataType.KEYWORD), () -> {
             List<Long> field = randomList(1, 10, () -> randomLong());
             BytesRef order = new BytesRef("DESC");
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.DATETIME, "field"),
-                    new TestCaseSupplier.TypedData(order, DataTypes.KEYWORD, "order").forceLiteral()
+                    new TestCaseSupplier.TypedData(field, DataType.DATETIME, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
                 "MvSort" + ElementType.LONG + "[field=Attribute[channel=0], order=false]",
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
             );
         }));
     }
 
     private static void doubles(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.DOUBLE, DataTypes.KEYWORD), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.DOUBLE, DataType.KEYWORD), () -> {
             List<Double> field = randomList(1, 10, () -> randomDouble());
             BytesRef order = new BytesRef("ASC");
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.DOUBLE, "field"),
-                    new TestCaseSupplier.TypedData(order, DataTypes.KEYWORD, "order").forceLiteral()
+                    new TestCaseSupplier.TypedData(field, DataType.DOUBLE, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
                 "MvSort" + ElementType.DOUBLE + "[field=Attribute[channel=0], order=true]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
             );
         }));
     }
 
     private static void bytesRefs(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.KEYWORD, DataTypes.KEYWORD), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataTypes.KEYWORD).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD, DataType.KEYWORD), () -> {
+            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.KEYWORD).value());
             BytesRef order = new BytesRef("DESC");
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.KEYWORD, "field"),
-                    new TestCaseSupplier.TypedData(order, DataTypes.KEYWORD, "order").forceLiteral()
+                    new TestCaseSupplier.TypedData(field, DataType.KEYWORD, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
                 "MvSort" + ElementType.BYTES_REF + "[field=Attribute[channel=0], order=false]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.TEXT, DataTypes.KEYWORD), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataTypes.TEXT).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT, DataType.KEYWORD), () -> {
+            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value());
             BytesRef order = new BytesRef("ASC");
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.TEXT, "field"),
-                    new TestCaseSupplier.TypedData(order, DataTypes.KEYWORD, "order").forceLiteral()
+                    new TestCaseSupplier.TypedData(field, DataType.TEXT, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
                 "MvSort" + ElementType.BYTES_REF + "[field=Attribute[channel=0], order=true]",
-                DataTypes.TEXT,
+                DataType.TEXT,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.KEYWORD), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataTypes.IP).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.KEYWORD), () -> {
+            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.IP).value());
             BytesRef order = new BytesRef("DESC");
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.IP, "field"),
-                    new TestCaseSupplier.TypedData(order, DataTypes.KEYWORD, "order").forceLiteral()
+                    new TestCaseSupplier.TypedData(field, DataType.IP, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
                 "MvSort" + ElementType.BYTES_REF + "[field=Attribute[channel=0], order=false]",
-                DataTypes.IP,
+                DataType.IP,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList())
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.VERSION, DataTypes.KEYWORD), () -> {
-            List<Object> field = randomList(1, 10, () -> randomLiteral(DataTypes.VERSION).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.VERSION, DataType.KEYWORD), () -> {
+            List<Object> field = randomList(1, 10, () -> randomLiteral(DataType.VERSION).value());
             BytesRef order = new BytesRef("ASC");
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field, DataTypes.VERSION, "field"),
-                    new TestCaseSupplier.TypedData(order, DataTypes.KEYWORD, "order").forceLiteral()
+                    new TestCaseSupplier.TypedData(field, DataType.VERSION, "field"),
+                    new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral()
                 ),
                 "MvSort" + ElementType.BYTES_REF + "[field=Attribute[channel=0], order=true]",
-                DataTypes.VERSION,
+                DataType.VERSION,
                 equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList())
             );
         }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java
index 5dd248323923f..c1d820cd93931 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
 import java.util.ArrayList;
@@ -41,27 +40,27 @@ public static Iterable<Object[]> parameters() {
         // longs(cases, "mv_sum", "MvSum", (size, values) -> equalTo(values.sum()));
         // unsignedLongAsBigInteger(cases, "mv_sum", "MvSum", (size, values) -> equalTo(values.sum()));
 
-        cases.add(arithmeticExceptionCase(DataTypes.INTEGER, () -> {
+        cases.add(arithmeticExceptionCase(DataType.INTEGER, () -> {
             List<Object> data = randomList(1, 10, () -> randomIntBetween(0, Integer.MAX_VALUE));
             data.add(Integer.MAX_VALUE);
             return data;
         }));
-        cases.add(arithmeticExceptionCase(DataTypes.INTEGER, () -> {
+        cases.add(arithmeticExceptionCase(DataType.INTEGER, () -> {
             List<Object> data = randomList(1, 10, () -> randomIntBetween(Integer.MIN_VALUE, 0));
             data.add(Integer.MIN_VALUE);
             return data;
         }));
-        cases.add(arithmeticExceptionCase(DataTypes.LONG, () -> {
+        cases.add(arithmeticExceptionCase(DataType.LONG, () -> {
             List<Object> data = randomList(1, 10, () -> randomLongBetween(0L, Long.MAX_VALUE));
             data.add(Long.MAX_VALUE);
             return data;
         }));
-        cases.add(arithmeticExceptionCase(DataTypes.LONG, () -> {
+        cases.add(arithmeticExceptionCase(DataType.LONG, () -> {
             List<Object> data = randomList(1, 10, () -> randomLongBetween(Long.MIN_VALUE, 0L));
             data.add(Long.MIN_VALUE);
             return data;
         }));
-        cases.add(arithmeticExceptionCase(DataTypes.UNSIGNED_LONG, () -> {
+        cases.add(arithmeticExceptionCase(DataType.UNSIGNED_LONG, () -> {
             List<Object> data = randomList(1, 10, ESTestCase::randomLong);
             data.add(asLongUnsigned(UNSIGNED_LONG_MAX));
             return data;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java
index 8d0e2a64d1643..e465d72555e4e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
 
@@ -34,36 +33,36 @@ public MvZipTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCase
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.KEYWORD, DataTypes.KEYWORD, DataTypes.KEYWORD), () -> {
-            List<Object> left = randomList(1, 3, () -> randomLiteral(DataTypes.KEYWORD).value());
-            List<Object> right = randomList(1, 3, () -> randomLiteral(DataTypes.KEYWORD).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD, DataType.KEYWORD, DataType.KEYWORD), () -> {
+            List<Object> left = randomList(1, 3, () -> randomLiteral(DataType.KEYWORD).value());
+            List<Object> right = randomList(1, 3, () -> randomLiteral(DataType.KEYWORD).value());
             String delim = randomAlphaOfLengthBetween(1, 1);
             List<BytesRef> expected = calculateExpected(left, right, delim);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(left, DataTypes.KEYWORD, "mvLeft"),
-                    new TestCaseSupplier.TypedData(right, DataTypes.KEYWORD, "mvRight"),
-                    new TestCaseSupplier.TypedData(delim, DataTypes.KEYWORD, "delim")
+                    new TestCaseSupplier.TypedData(left, DataType.KEYWORD, "mvLeft"),
+                    new TestCaseSupplier.TypedData(right, DataType.KEYWORD, "mvRight"),
+                    new TestCaseSupplier.TypedData(delim, DataType.KEYWORD, "delim")
                 ),
                 "MvZipEvaluator[leftField=Attribute[channel=0], rightField=Attribute[channel=1], delim=Attribute[channel=2]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(expected.size() == 1 ? expected.iterator().next() : expected)
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.TEXT, DataTypes.TEXT, DataTypes.TEXT), () -> {
-            List<Object> left = randomList(1, 10, () -> randomLiteral(DataTypes.TEXT).value());
-            List<Object> right = randomList(1, 10, () -> randomLiteral(DataTypes.TEXT).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT, DataType.TEXT, DataType.TEXT), () -> {
+            List<Object> left = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value());
+            List<Object> right = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value());
             String delim = randomAlphaOfLengthBetween(1, 1);
             List<BytesRef> expected = calculateExpected(left, right, delim);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(left, DataTypes.TEXT, "mvLeft"),
-                    new TestCaseSupplier.TypedData(right, DataTypes.TEXT, "mvRight"),
-                    new TestCaseSupplier.TypedData(delim, DataTypes.TEXT, "delim")
+                    new TestCaseSupplier.TypedData(left, DataType.TEXT, "mvLeft"),
+                    new TestCaseSupplier.TypedData(right, DataType.TEXT, "mvRight"),
+                    new TestCaseSupplier.TypedData(delim, DataType.TEXT, "delim")
                 ),
                 "MvZipEvaluator[leftField=Attribute[channel=0], rightField=Attribute[channel=1], delim=Attribute[channel=2]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(expected.size() == 1 ? expected.iterator().next() : expected)
             );
         }));
@@ -73,7 +72,7 @@ public static Iterable<Object[]> parameters() {
 
     @Override
     protected DataType expectedType(List<DataType> argTypes) {
-        return DataTypes.KEYWORD;
+        return DataType.KEYWORD;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
index e884ba9400206..77a129c8eb625 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
@@ -59,29 +58,29 @@ public static Iterable<Object[]> parameters() {
         builder.expectBoolean(booleans -> booleans.filter(v -> v != null).findFirst());
         suppliers.addAll(builder.suppliers());
         addSpatialCombinations(suppliers);
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.IP), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.IP), () -> {
             var first = randomBoolean() ? null : EsqlDataTypeConverter.stringToIP(NetworkAddress.format(randomIp(true)));
             var second = EsqlDataTypeConverter.stringToIP(NetworkAddress.format(randomIp(true)));
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(first, DataTypes.IP, "first"),
-                    new TestCaseSupplier.TypedData(second, DataTypes.IP, "second")
+                    new TestCaseSupplier.TypedData(first, DataType.IP, "first"),
+                    new TestCaseSupplier.TypedData(second, DataType.IP, "second")
                 ),
                 "CoalesceEvaluator[values=[Attribute[channel=0], Attribute[channel=1]]]",
-                DataTypes.IP,
+                DataType.IP,
                 equalTo(first == null ? second : first)
             );
         }));
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.DATETIME, DataTypes.DATETIME), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.DATETIME, DataType.DATETIME), () -> {
             Long firstDate = randomBoolean() ? null : ZonedDateTime.parse("2023-12-04T10:15:30Z").toInstant().toEpochMilli();
             Long secondDate = ZonedDateTime.parse("2023-12-05T10:45:00Z").toInstant().toEpochMilli();
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(firstDate, DataTypes.DATETIME, "first"),
-                    new TestCaseSupplier.TypedData(secondDate, DataTypes.DATETIME, "second")
+                    new TestCaseSupplier.TypedData(firstDate, DataType.DATETIME, "first"),
+                    new TestCaseSupplier.TypedData(secondDate, DataType.DATETIME, "second")
                 ),
                 "CoalesceEvaluator[values=[Attribute[channel=0], Attribute[channel=1]]]",
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 equalTo(firstDate == null ? secondDate : firstDate)
             );
         }));
@@ -90,7 +89,7 @@ public static Iterable<Object[]> parameters() {
     }
 
     protected static void addSpatialCombinations(List<TestCaseSupplier> suppliers) {
-        for (DataType dataType : List.of(DataTypes.GEO_POINT, DataTypes.GEO_SHAPE, DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE)) {
+        for (DataType dataType : List.of(DataType.GEO_POINT, DataType.GEO_SHAPE, DataType.CARTESIAN_POINT, DataType.CARTESIAN_SHAPE)) {
             TestCaseSupplier.TypedDataSupplier leftDataSupplier = SpatialRelatesFunctionTestCase.testCaseSupplier(dataType);
             TestCaseSupplier.TypedDataSupplier rightDataSupplier = SpatialRelatesFunctionTestCase.testCaseSupplier(dataType);
             suppliers.add(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java
index b593ad4d3aed4..299b66433dcd0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -36,11 +35,11 @@ public IsNotNullTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> test
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType type : DataTypes.types()) {
+        for (DataType type : DataType.types()) {
             if (false == EsqlDataTypes.isRepresentable(type)) {
                 continue;
             }
-            if (type != DataTypes.NULL) {
+            if (type != DataType.NULL) {
                 suppliers.add(
                     new TestCaseSupplier(
                         "non-null " + type.typeName(),
@@ -48,7 +47,7 @@ public static Iterable<Object[]> parameters() {
                         () -> new TestCaseSupplier.TestCase(
                             List.of(new TestCaseSupplier.TypedData(randomLiteral(type).value(), type, "v")),
                             "IsNotNullEvaluator[field=Attribute[channel=0]]",
-                            DataTypes.BOOLEAN,
+                            DataType.BOOLEAN,
                             equalTo(true)
                         )
                     )
@@ -61,7 +60,7 @@ public static Iterable<Object[]> parameters() {
                     () -> new TestCaseSupplier.TestCase(
                         List.of(new TestCaseSupplier.TypedData(null, type, "v")),
                         "IsNotNullEvaluator[field=Attribute[channel=0]]",
-                        DataTypes.BOOLEAN,
+                        DataType.BOOLEAN,
                         equalTo(false)
                     )
                 )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java
index adeca03406ead..606e9598bda63 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -36,11 +35,11 @@ public IsNullTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCas
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType type : DataTypes.types()) {
+        for (DataType type : DataType.types()) {
             if (false == EsqlDataTypes.isRepresentable(type)) {
                 continue;
             }
-            if (type != DataTypes.NULL) {
+            if (type != DataType.NULL) {
                 suppliers.add(
                     new TestCaseSupplier(
                         "non-null " + type.typeName(),
@@ -48,7 +47,7 @@ public static Iterable<Object[]> parameters() {
                         () -> new TestCaseSupplier.TestCase(
                             List.of(new TestCaseSupplier.TypedData(randomLiteral(type).value(), type, "v")),
                             "IsNullEvaluator[field=Attribute[channel=0]]",
-                            DataTypes.BOOLEAN,
+                            DataType.BOOLEAN,
                             equalTo(false)
                         )
                     )
@@ -61,7 +60,7 @@ public static Iterable<Object[]> parameters() {
                     () -> new TestCaseSupplier.TestCase(
                         List.of(new TestCaseSupplier.TypedData(null, type, "v")),
                         "IsNullEvaluator[field=Attribute[channel=0]]",
-                        DataTypes.BOOLEAN,
+                        DataType.BOOLEAN,
                         equalTo(true)
                     )
                 )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsTests.java
index b29a952f4c792..d65ad5a2b961c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContainsTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -30,9 +29,9 @@ public SpatialContainsTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        DataType[] geoDataTypes = { DataTypes.GEO_POINT, DataTypes.GEO_SHAPE };
+        DataType[] geoDataTypes = { DataType.GEO_POINT, DataType.GEO_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, geoDataTypes);
-        DataType[] cartesianDataTypes = { DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE };
+        DataType[] cartesianDataTypes = { DataType.CARTESIAN_POINT, DataType.CARTESIAN_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, cartesianDataTypes);
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), SpatialContainsTests::typeErrorMessage)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointTests.java
index 999fc695fd8fe..b3feac5619c16 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjointTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -30,9 +29,9 @@ public SpatialDisjointTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        DataType[] geoDataTypes = { DataTypes.GEO_POINT, DataTypes.GEO_SHAPE };
+        DataType[] geoDataTypes = { DataType.GEO_POINT, DataType.GEO_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, geoDataTypes);
-        DataType[] cartesianDataTypes = { DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE };
+        DataType[] cartesianDataTypes = { DataType.CARTESIAN_POINT, DataType.CARTESIAN_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, cartesianDataTypes);
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), SpatialDisjointTests::typeErrorMessage)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsTests.java
index 19d43f42a9555..ccf94bf6d2760 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersectsTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -30,9 +29,9 @@ public SpatialIntersectsTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCa
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        DataType[] geoDataTypes = { DataTypes.GEO_POINT, DataTypes.GEO_SHAPE };
+        DataType[] geoDataTypes = { DataType.GEO_POINT, DataType.GEO_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, geoDataTypes);
-        DataType[] cartesianDataTypes = { DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE };
+        DataType[] cartesianDataTypes = { DataType.CARTESIAN_POINT, DataType.CARTESIAN_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, cartesianDataTypes);
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), SpatialIntersectsTests::typeErrorMessage)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunctionTestCase.java
index c689adfe50b29..9929971c48613 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunctionTestCase.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -65,7 +64,7 @@ protected static void addSpatialCombinations(List<TestCaseSupplier> suppliers, D
                             leftDataSupplier,
                             rightDataSupplier,
                             SpatialRelatesFunctionTestCase::spatialEvaluatorString,
-                            DataTypes.BOOLEAN,
+                            DataType.BOOLEAN,
                             (l, r) -> expected(l, leftType, r, rightType)
                         )
                     );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinTests.java
index 3dde6a1f83a3f..5f48cfcd6d701 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithinTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -30,9 +29,9 @@ public SpatialWithinTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase>
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        DataType[] geoDataTypes = { DataTypes.GEO_POINT, DataTypes.GEO_SHAPE };
+        DataType[] geoDataTypes = { DataType.GEO_POINT, DataType.GEO_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, geoDataTypes);
-        DataType[] cartesianDataTypes = { DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_SHAPE };
+        DataType[] cartesianDataTypes = { DataType.CARTESIAN_POINT, DataType.CARTESIAN_SHAPE };
         SpatialRelatesFunctionTestCase.addSpatialCombinations(suppliers, cartesianDataTypes);
         return parameterSuppliersFromTypedData(
             errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), SpatialWithinTests::typeErrorMessage)
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java
index 012ee982d3da6..b466ffe1e92f1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StXTests.java
@@ -21,7 +21,7 @@
 import java.util.List;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED;
 
 @FunctionName("st_x")
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java
index 776a1a461b10f..1f3639bf1ecb4 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StYTests.java
@@ -21,7 +21,7 @@
 import java.util.List;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
 import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED;
 
 @FunctionName("st_y")
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
index 9c8856280aa87..f46ae25fddfc7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -50,18 +49,18 @@ public static Iterable<Object[]> parameters() {
         for (int length = 4; length < 100; length++) {
             suppliers(suppliers, length);
         }
-        Set<DataType> supported = Set.of(DataTypes.NULL, DataTypes.KEYWORD, DataTypes.TEXT);
+        Set<DataType> supported = Set.of(DataType.NULL, DataType.KEYWORD, DataType.TEXT);
         List<Set<DataType>> supportedPerPosition = List.of(supported, supported);
-        for (DataType lhs : DataTypes.types()) {
-            if (lhs == DataTypes.NULL || EsqlDataTypes.isRepresentable(lhs) == false) {
+        for (DataType lhs : DataType.types()) {
+            if (lhs == DataType.NULL || EsqlDataTypes.isRepresentable(lhs) == false) {
                 continue;
             }
-            for (DataType rhs : DataTypes.types()) {
-                if (rhs == DataTypes.NULL || EsqlDataTypes.isRepresentable(rhs) == false) {
+            for (DataType rhs : DataType.types()) {
+                if (rhs == DataType.NULL || EsqlDataTypes.isRepresentable(rhs) == false) {
                     continue;
                 }
-                boolean lhsIsString = lhs == DataTypes.KEYWORD || lhs == DataTypes.TEXT;
-                boolean rhsIsString = rhs == DataTypes.KEYWORD || rhs == DataTypes.TEXT;
+                boolean lhsIsString = lhs == DataType.KEYWORD || lhs == DataType.TEXT;
+                boolean rhsIsString = rhs == DataType.KEYWORD || rhs == DataType.TEXT;
                 if (lhsIsString && rhsIsString) {
                     continue;
                 }
@@ -74,8 +73,8 @@ public static Iterable<Object[]> parameters() {
 
     private static void suppliers(List<TestCaseSupplier> suppliers, int length) {
         if (length > 3) {
-            suppliers.add(supplier("ascii", DataTypes.KEYWORD, length, () -> randomAlphaOfLengthBetween(1, 10)));
-            suppliers.add(supplier("unicode", DataTypes.TEXT, length, () -> randomRealisticUnicodeOfLengthBetween(1, 10)));
+            suppliers.add(supplier("ascii", DataType.KEYWORD, length, () -> randomAlphaOfLengthBetween(1, 10)));
+            suppliers.add(supplier("unicode", DataType.TEXT, length, () -> randomRealisticUnicodeOfLengthBetween(1, 10)));
         } else {
             add(suppliers, "ascii", length, () -> randomAlphaOfLengthBetween(1, 10));
             add(suppliers, "unicode", length, () -> randomRealisticUnicodeOfLengthBetween(1, 10));
@@ -97,14 +96,14 @@ private static TestCaseSupplier supplier(String name, DataType type, int length,
                 expectedToString += "Attribute[channel=" + v + "]";
             }
             expectedToString += "]]";
-            return new TestCaseSupplier.TestCase(values, expectedToString, DataTypes.KEYWORD, equalTo(new BytesRef(expectedValue)));
+            return new TestCaseSupplier.TestCase(values, expectedToString, DataType.KEYWORD, equalTo(new BytesRef(expectedValue)));
         });
     }
 
     private static void add(List<TestCaseSupplier> suppliers, String name, int length, Supplier<String> valueSupplier) {
         Map<Integer, List<List<DataType>>> permutations = new HashMap<Integer, List<List<DataType>>>();
-        List<DataType> supportedDataTypes = List.of(DataTypes.KEYWORD, DataTypes.TEXT);
-        permutations.put(0, List.of(List.of(DataTypes.KEYWORD), List.of(DataTypes.TEXT)));
+        List<DataType> supportedDataTypes = List.of(DataType.KEYWORD, DataType.TEXT);
+        permutations.put(0, List.of(List.of(DataType.KEYWORD), List.of(DataType.TEXT)));
         for (int v = 0; v < length - 1; v++) {
             List<List<DataType>> current = permutations.get(v);
             List<List<DataType>> next = new ArrayList<>();
@@ -133,7 +132,7 @@ private static void add(List<TestCaseSupplier> suppliers, String name, int lengt
                     expectedToString += "Attribute[channel=" + v + "]";
                 }
                 expectedToString += "]]";
-                return new TestCaseSupplier.TestCase(values, expectedToString, DataTypes.KEYWORD, equalTo(new BytesRef(expectedValue)));
+                return new TestCaseSupplier.TestCase(values, expectedToString, DataType.KEYWORD, equalTo(new BytesRef(expectedValue)));
             }));
         }
 
@@ -189,7 +188,7 @@ public void testSomeConstant() {
     private void testOversized(int totalLen, List<Expression> mix, List<Object> fieldValues) {
         for (int len; totalLen < Concat.MAX_CONCAT_LENGTH; totalLen += len) {
             len = randomIntBetween(1, (int) Concat.MAX_CONCAT_LENGTH);
-            mix.add(new Literal(Source.EMPTY, new BytesRef(randomAlphaOfLength(len)), DataTypes.KEYWORD));
+            mix.add(new Literal(Source.EMPTY, new BytesRef(randomAlphaOfLength(len)), DataType.KEYWORD));
         }
         Expression expression = build(testCase.getSource(), mix);
         Exception e = expectThrows(EsqlClientException.class, () -> {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
index 6443d739e64c5..863243a352bb0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -37,11 +37,11 @@ public static Iterable<Object[]> parameters() {
             String suffix = "";
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.KEYWORD, "suffix")
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.KEYWORD, "suffix")
                 ),
                 "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(str.endsWith(suffix))
             );
         }));
@@ -50,11 +50,11 @@ public static Iterable<Object[]> parameters() {
             String suffix = randomAlphaOfLength(5);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.KEYWORD, "suffix")
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.KEYWORD, "suffix")
                 ),
                 "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(str.endsWith(suffix))
             );
         }));
@@ -65,11 +65,11 @@ public static Iterable<Object[]> parameters() {
 
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.KEYWORD, "suffix")
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.KEYWORD, "suffix")
                 ),
                 "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(str.endsWith(suffix))
             );
         }));
@@ -80,11 +80,11 @@ public static Iterable<Object[]> parameters() {
 
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.KEYWORD, "suffix")
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.KEYWORD, "suffix")
                 ),
                 "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(str.endsWith(suffix))
             );
         }));
@@ -95,11 +95,11 @@ public static Iterable<Object[]> parameters() {
 
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.KEYWORD, "suffix")
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.KEYWORD, "suffix")
                 ),
                 "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(str.endsWith(suffix))
             );
         }));
@@ -110,11 +110,11 @@ public static Iterable<Object[]> parameters() {
 
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.TEXT, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataTypes.TEXT, "suffix")
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.TEXT, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(suffix), DataType.TEXT, "suffix")
                 ),
                 "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(str.endsWith(suffix))
             );
         }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java
index 291b0c5d6ce3c..e6a5d30d0fa53 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -37,132 +37,132 @@ public LeftTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseS
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        suppliers.add(new TestCaseSupplier("empty string", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("empty string", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             int length = between(-64, 64);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(""), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(""), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(""))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier("ascii", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("ascii", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomAlphaOfLengthBetween(1, 64);
             int length = between(1, text.length());
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(unicodeLeftSubstring(text, length)))
             );
         }));
-        suppliers.add(new TestCaseSupplier("ascii longer than string", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("ascii longer than string", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomAlphaOfLengthBetween(1, 64);
             int length = between(text.length(), 128);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(text))
             );
         }));
-        suppliers.add(new TestCaseSupplier("ascii zero length", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("ascii zero length", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomAlphaOfLengthBetween(1, 64);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(0, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(0, DataType.INTEGER, "length")
                 ),
                 "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(""))
             );
         }));
-        suppliers.add(new TestCaseSupplier("ascii negative length", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("ascii negative length", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomAlphaOfLengthBetween(1, 64);
             int length = between(-128, -1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(""))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier("unicode", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("unicode", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomUnicodeOfLengthBetween(1, 64);
             int length = between(1, text.length());
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(unicodeLeftSubstring(text, length)))
             );
         }));
-        suppliers.add(new TestCaseSupplier("unicode longer than string", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("unicode longer than string", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomUnicodeOfLengthBetween(1, 64);
             int length = between(text.length(), 128);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(text))
             );
         }));
-        suppliers.add(new TestCaseSupplier("unicode zero length", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("unicode zero length", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomUnicodeOfLengthBetween(1, 64);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(0, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(0, DataType.INTEGER, "length")
                 ),
                 "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(""))
             );
         }));
-        suppliers.add(new TestCaseSupplier("unicode negative length", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("unicode negative length", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomUnicodeOfLengthBetween(1, 64);
             int length = between(-128, -1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(""))
             );
         }));
-        suppliers.add(new TestCaseSupplier("ascii as text input", List.of(DataTypes.TEXT, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("ascii as text input", List.of(DataType.TEXT, DataType.INTEGER), () -> {
             String text = randomAlphaOfLengthBetween(1, 64);
             int length = between(1, text.length());
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.TEXT, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.TEXT, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(unicodeLeftSubstring(text, length)))
             );
         }));
@@ -201,7 +201,7 @@ public void testUnicode() {
     private String process(String str, int length) {
         try (
             EvalOperator.ExpressionEvaluator eval = evaluator(
-                new Left(Source.EMPTY, field("str", DataTypes.KEYWORD), new Literal(Source.EMPTY, length, DataTypes.INTEGER))
+                new Left(Source.EMPTY, field("str", DataType.KEYWORD), new Literal(Source.EMPTY, length, DataType.INTEGER))
             ).get(driverContext());
             Block block = eval.eval(row(List.of(new BytesRef(str))))
         ) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java
index 98ff38b48a7cf..81fcc118ade05 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java
@@ -14,7 +14,7 @@
 import org.apache.lucene.util.UnicodeUtil;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -33,12 +33,12 @@ public LengthTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCas
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> cases = new ArrayList<>();
-        cases.addAll(List.of(new TestCaseSupplier("length basic test", List.of(DataTypes.KEYWORD), () -> {
+        cases.addAll(List.of(new TestCaseSupplier("length basic test", List.of(DataType.KEYWORD), () -> {
             BytesRef value = new BytesRef(randomAlphaOfLength(between(0, 10000)));
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(value, DataTypes.KEYWORD, "f")),
+                List.of(new TestCaseSupplier.TypedData(value, DataType.KEYWORD, "f")),
                 "LengthEvaluator[val=Attribute[channel=0]]",
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 equalTo(UnicodeUtil.codePointCount(value))
             );
         })));
@@ -56,21 +56,21 @@ private static List<TestCaseSupplier> makeTestCases(String title, Supplier<Strin
         return List.of(
             new TestCaseSupplier(
                 title + " with keyword",
-                List.of(DataTypes.KEYWORD),
+                List.of(DataType.KEYWORD),
                 () -> new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataTypes.KEYWORD, "f")),
+                    List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.KEYWORD, "f")),
                     "LengthEvaluator[val=Attribute[channel=0]]",
-                    DataTypes.INTEGER,
+                    DataType.INTEGER,
                     equalTo(expectedLength)
                 )
             ),
             new TestCaseSupplier(
                 title + " with text",
-                List.of(DataTypes.TEXT),
+                List.of(DataType.TEXT),
                 () -> new TestCaseSupplier.TestCase(
-                    List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataTypes.TEXT, "f")),
+                    List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.TEXT, "f")),
                     "LengthEvaluator[val=Attribute[channel=0]]",
-                    DataTypes.INTEGER,
+                    DataType.INTEGER,
                     equalTo(expectedLength)
                 )
             )
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
index fff74b8af7a15..011252a3f7e14 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -36,7 +35,7 @@ public LocateTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCas
         this.testCase = testCaseSupplier.get();
     }
 
-    private static final DataType[] STRING_TYPES = new DataType[] { DataTypes.KEYWORD, DataTypes.TEXT };
+    private static final DataType[] STRING_TYPES = new DataType[] { DataType.KEYWORD, DataType.TEXT };
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
@@ -147,8 +146,8 @@ private static TestCaseSupplier supplier(String str, String substr, @Nullable In
 
         return new TestCaseSupplier(
             name,
-            types(DataTypes.KEYWORD, DataTypes.KEYWORD, start != null),
-            () -> testCase(DataTypes.KEYWORD, DataTypes.KEYWORD, str, substr, start, expectedValue)
+            types(DataType.KEYWORD, DataType.KEYWORD, start != null),
+            () -> testCase(DataType.KEYWORD, DataType.KEYWORD, str, substr, start, expectedValue)
         );
     }
 
@@ -186,7 +185,7 @@ private static List<DataType> types(DataType firstType, DataType secondType, boo
         types.add(firstType);
         types.add(secondType);
         if (hasStart) {
-            types.add(DataTypes.INTEGER);
+            types.add(DataType.INTEGER);
         }
         return types;
     }
@@ -203,8 +202,8 @@ private static TestCaseSupplier.TestCase testCase(
         values.add(new TestCaseSupplier.TypedData(str == null ? null : new BytesRef(str), strType, "str"));
         values.add(new TestCaseSupplier.TypedData(substr == null ? null : new BytesRef(substr), substrType, "substr"));
         if (start != null) {
-            values.add(new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"));
+            values.add(new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"));
         }
-        return new TestCaseSupplier.TestCase(values, expectedToString(start != null), DataTypes.INTEGER, equalTo(expectedValue));
+        return new TestCaseSupplier.TestCase(values, expectedToString(start != null), DataType.INTEGER, equalTo(expectedValue));
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
index 57883e0b84067..e1bcc519840be 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -51,15 +50,15 @@ static Iterable<Object[]> parameters(Function<String, String> escapeString, Supp
         cases.add(
             new TestCaseSupplier(
                 "null",
-                List.of(DataTypes.NULL, DataTypes.KEYWORD, DataTypes.BOOLEAN),
+                List.of(DataType.NULL, DataType.KEYWORD, DataType.BOOLEAN),
                 () -> new TestCaseSupplier.TestCase(
                     List.of(
-                        new TestCaseSupplier.TypedData(null, DataTypes.NULL, "e"),
-                        new TestCaseSupplier.TypedData(new BytesRef(randomAlphaOfLength(10)), DataTypes.KEYWORD, "pattern").forceLiteral(),
-                        new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "caseInsensitive").forceLiteral()
+                        new TestCaseSupplier.TypedData(null, DataType.NULL, "e"),
+                        new TestCaseSupplier.TypedData(new BytesRef(randomAlphaOfLength(10)), DataType.KEYWORD, "pattern").forceLiteral(),
+                        new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "caseInsensitive").forceLiteral()
                     ),
                     "LiteralsEvaluator[lit=null]",
-                    DataTypes.BOOLEAN,
+                    DataType.BOOLEAN,
                     nullValue()
                 )
             )
@@ -70,8 +69,8 @@ static Iterable<Object[]> parameters(Function<String, String> escapeString, Supp
         casesForString(cases, "3 bytes, 1 code point", () -> "☕", false, escapeString, optionalPattern);
         casesForString(cases, "6 bytes, 2 code points", () -> "❗️", false, escapeString, optionalPattern);
         casesForString(cases, "100 random code points", () -> randomUnicodeOfCodepointLength(100), true, escapeString, optionalPattern);
-        for (DataType type : DataTypes.types()) {
-            if (type == DataTypes.KEYWORD || type == DataTypes.TEXT || type == DataTypes.NULL) {
+        for (DataType type : DataType.types()) {
+            if (type == DataType.KEYWORD || type == DataType.TEXT || type == DataType.NULL) {
                 continue;
             }
             if (EsqlDataTypes.isRepresentable(type) == false) {
@@ -79,13 +78,13 @@ static Iterable<Object[]> parameters(Function<String, String> escapeString, Supp
             }
             cases.add(
                 new TestCaseSupplier(
-                    List.of(type, DataTypes.KEYWORD, DataTypes.BOOLEAN),
+                    List.of(type, DataType.KEYWORD, DataType.BOOLEAN),
                     () -> TestCaseSupplier.TestCase.typeError(
                         List.of(
                             new TestCaseSupplier.TypedData(randomLiteral(type).value(), type, "e"),
-                            new TestCaseSupplier.TypedData(new BytesRef(randomAlphaOfLength(10)), DataTypes.KEYWORD, "pattern")
+                            new TestCaseSupplier.TypedData(new BytesRef(randomAlphaOfLength(10)), DataType.KEYWORD, "pattern")
                                 .forceLiteral(),
-                            new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "caseInsensitive").forceLiteral()
+                            new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "caseInsensitive").forceLiteral()
                         ),
                         "argument of [] must be [string], found value [e] type [" + type.typeName() + "]"
                     )
@@ -127,17 +126,17 @@ private static void casesForString(
     }
 
     private static void cases(List<TestCaseSupplier> cases, String title, Supplier<TextAndPattern> textAndPattern, boolean expected) {
-        for (DataType type : new DataType[] { DataTypes.KEYWORD, DataTypes.TEXT }) {
-            cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, type, DataTypes.BOOLEAN), () -> {
+        for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT }) {
+            cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, type, DataType.BOOLEAN), () -> {
                 TextAndPattern v = textAndPattern.get();
                 return new TestCaseSupplier.TestCase(
                     List.of(
                         new TestCaseSupplier.TypedData(new BytesRef(v.text), type, "e"),
                         new TestCaseSupplier.TypedData(new BytesRef(v.pattern), type, "pattern").forceLiteral(),
-                        new TestCaseSupplier.TypedData(false, DataTypes.BOOLEAN, "caseInsensitive").forceLiteral()
+                        new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "caseInsensitive").forceLiteral()
                     ),
                     startsWith("AutomataMatchEvaluator[input=Attribute[channel=0], pattern=digraph Automaton {\n"),
-                    DataTypes.BOOLEAN,
+                    DataType.BOOLEAN,
                     equalTo(expected)
                 );
             }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
index f8cee3c2dd42f..bfadf66f7f5cc 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -33,16 +32,16 @@ public ReplaceTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCa
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        for (DataType strType : DataTypes.types()) {
-            if (DataTypes.isString(strType) == false) {
+        for (DataType strType : DataType.types()) {
+            if (DataType.isString(strType) == false) {
                 continue;
             }
-            for (DataType oldStrType : DataTypes.types()) {
-                if (DataTypes.isString(oldStrType) == false) {
+            for (DataType oldStrType : DataType.types()) {
+                if (DataType.isString(oldStrType) == false) {
                     continue;
                 }
-                for (DataType newStrType : DataTypes.types()) {
-                    if (DataTypes.isString(newStrType) == false) {
+                for (DataType newStrType : DataType.types()) {
+                    if (DataType.isString(newStrType) == false) {
                         continue;
                     }
                     suppliers.add(new TestCaseSupplier(List.of(strType, oldStrType, newStrType), () -> {
@@ -79,18 +78,18 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        suppliers.add(new TestCaseSupplier("syntax error", List.of(DataTypes.KEYWORD, DataTypes.KEYWORD, DataTypes.KEYWORD), () -> {
+        suppliers.add(new TestCaseSupplier("syntax error", List.of(DataType.KEYWORD, DataType.KEYWORD, DataType.KEYWORD), () -> {
             String text = randomAlphaOfLength(10);
             String invalidRegex = "[";
             String newStr = randomAlphaOfLength(5);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(invalidRegex), DataTypes.KEYWORD, "oldStr"),
-                    new TestCaseSupplier.TypedData(new BytesRef(newStr), DataTypes.KEYWORD, "newStr")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(invalidRegex), DataType.KEYWORD, "oldStr"),
+                    new TestCaseSupplier.TypedData(new BytesRef(newStr), DataType.KEYWORD, "newStr")
                 ),
                 "ReplaceEvaluator[str=Attribute[channel=0], regex=Attribute[channel=1], newStr=Attribute[channel=2]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(null)
             ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
                 .withWarning(
@@ -110,8 +109,8 @@ public static Iterable<Object[]> parameters() {
     private static TestCaseSupplier fixedCase(String name, String str, String oldStr, String newStr, String result) {
         return new TestCaseSupplier(
             name,
-            List.of(DataTypes.KEYWORD, DataTypes.KEYWORD, DataTypes.KEYWORD),
-            () -> testCase(DataTypes.KEYWORD, DataTypes.KEYWORD, DataTypes.KEYWORD, str, oldStr, newStr, result)
+            List.of(DataType.KEYWORD, DataType.KEYWORD, DataType.KEYWORD),
+            () -> testCase(DataType.KEYWORD, DataType.KEYWORD, DataType.KEYWORD, str, oldStr, newStr, result)
         );
     }
 
@@ -131,7 +130,7 @@ private static TestCaseSupplier.TestCase testCase(
                 new TestCaseSupplier.TypedData(new BytesRef(newStr), newStrType, "newStr")
             ),
             "ReplaceEvaluator[str=Attribute[channel=0], regex=Attribute[channel=1], newStr=Attribute[channel=2]]",
-            DataTypes.KEYWORD,
+            DataType.KEYWORD,
             equalTo(new BytesRef(result))
         );
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java
index a925410197aee..599ab51995217 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -37,132 +37,132 @@ public RightTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCase
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        suppliers.add(new TestCaseSupplier("empty string", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("empty string", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             int length = between(-64, 64);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(""), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(""), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(""))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier("ascii", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("ascii", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomAlphaOfLengthBetween(1, 64);
             int length = between(1, text.length());
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(unicodeRightSubstring(text, length)))
             );
         }));
-        suppliers.add(new TestCaseSupplier("ascii longer than string", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("ascii longer than string", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomAlphaOfLengthBetween(1, 64);
             int length = between(text.length(), 128);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(text))
             );
         }));
-        suppliers.add(new TestCaseSupplier("ascii zero length", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("ascii zero length", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomAlphaOfLengthBetween(1, 64);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(0, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(0, DataType.INTEGER, "length")
                 ),
                 "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(""))
             );
         }));
-        suppliers.add(new TestCaseSupplier("ascii negative length", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("ascii negative length", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomAlphaOfLengthBetween(1, 64);
             int length = between(-128, -1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(""))
             );
         }));
 
-        suppliers.add(new TestCaseSupplier("unicode", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("unicode", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomUnicodeOfLengthBetween(1, 64);
             int length = between(1, text.length());
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(unicodeRightSubstring(text, length)))
             );
         }));
-        suppliers.add(new TestCaseSupplier("unicode longer than string", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("unicode longer than string", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomUnicodeOfLengthBetween(1, 64);
             int length = between(text.length(), 128);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(text))
             );
         }));
-        suppliers.add(new TestCaseSupplier("unicode zero length", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("unicode zero length", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomUnicodeOfLengthBetween(1, 64);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(0, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(0, DataType.INTEGER, "length")
                 ),
                 "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(""))
             );
         }));
-        suppliers.add(new TestCaseSupplier("unicode negative length", List.of(DataTypes.KEYWORD, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("unicode negative length", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
             String text = randomUnicodeOfLengthBetween(1, 64);
             int length = between(-128, -1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(""))
             );
         }));
-        suppliers.add(new TestCaseSupplier("ascii as text", List.of(DataTypes.TEXT, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier("ascii as text", List.of(DataType.TEXT, DataType.INTEGER), () -> {
             String text = randomAlphaOfLengthBetween(1, 64);
             int length = between(1, text.length());
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.TEXT, "str"),
-                    new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "length")
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.TEXT, "str"),
+                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length")
                 ),
                 "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(new BytesRef(unicodeRightSubstring(text, length)))
             );
         }));
@@ -202,7 +202,7 @@ public void testUnicode() {
     private String process(String str, int length) {
         try (
             EvalOperator.ExpressionEvaluator eval = evaluator(
-                new Right(Source.EMPTY, field("str", DataTypes.KEYWORD), new Literal(Source.EMPTY, length, DataTypes.INTEGER))
+                new Right(Source.EMPTY, field("str", DataType.KEYWORD), new Literal(Source.EMPTY, length, DataType.INTEGER))
             ).get(driverContext());
             Block block = eval.eval(row(List.of(new BytesRef(str))))
         ) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java
index d809c8ebc9f53..47e48df90007e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -43,7 +42,7 @@ public SplitTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCase
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        List<DataType> supportedDataTyes = List.of(DataTypes.KEYWORD, DataTypes.TEXT);
+        List<DataType> supportedDataTyes = List.of(DataType.KEYWORD, DataType.TEXT);
         for (DataType sType : supportedDataTyes) {
             for (DataType dType : supportedDataTyes) {
                 suppliers.add(new TestCaseSupplier("split test " + sType.toString() + " " + dType.toString(), List.of(sType, dType), () -> {
@@ -59,7 +58,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef(delimiter), dType, "delim")
                         ),
                         "SplitVariableEvaluator[str=Attribute[channel=0], delim=Attribute[channel=1]]",
-                        DataTypes.KEYWORD,
+                        DataType.KEYWORD,
                         equalTo(strings.size() == 1 ? strings.get(0) : strings)
                     );
                 }));
@@ -77,7 +76,7 @@ public void testConstantDelimiter() {
         DriverContext driverContext = driverContext();
         try (
             EvalOperator.ExpressionEvaluator eval = evaluator(
-                new Split(Source.EMPTY, field("str", DataTypes.KEYWORD), new Literal(Source.EMPTY, new BytesRef(":"), DataTypes.KEYWORD))
+                new Split(Source.EMPTY, field("str", DataType.KEYWORD), new Literal(Source.EMPTY, new BytesRef(":"), DataType.KEYWORD))
             ).get(driverContext)
         ) {
             /*
@@ -105,8 +104,8 @@ public void testTooLongConstantDelimiter() {
             () -> evaluator(
                 new Split(
                     Source.EMPTY,
-                    field("str", DataTypes.KEYWORD),
-                    new Literal(Source.EMPTY, new BytesRef(delimiter), DataTypes.KEYWORD)
+                    field("str", DataType.KEYWORD),
+                    new Literal(Source.EMPTY, new BytesRef(delimiter), DataType.KEYWORD)
                 )
             ).get(driverContext)
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
index 8b98cbd0f8ca7..f0c51a9b22e55 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -38,11 +38,11 @@ public static Iterable<Object[]> parameters() {
             }
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.KEYWORD, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(prefix), DataTypes.KEYWORD, "prefix")
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(prefix), DataType.KEYWORD, "prefix")
                 ),
                 "StartsWithEvaluator[str=Attribute[channel=0], prefix=Attribute[channel=1]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(str.startsWith(prefix))
             );
         }), new TestCaseSupplier("Starts with basic test with text args", () -> {
@@ -53,11 +53,11 @@ public static Iterable<Object[]> parameters() {
             }
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(new BytesRef(str), DataTypes.TEXT, "str"),
-                    new TestCaseSupplier.TypedData(new BytesRef(prefix), DataTypes.TEXT, "prefix")
+                    new TestCaseSupplier.TypedData(new BytesRef(str), DataType.TEXT, "str"),
+                    new TestCaseSupplier.TypedData(new BytesRef(prefix), DataType.TEXT, "prefix")
                 ),
                 "StartsWithEvaluator[str=Attribute[channel=0], prefix=Attribute[channel=1]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(str.startsWith(prefix))
             );
         })));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
index c2df6f61c1d25..c1a49455d9d83 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -41,62 +41,54 @@ public static Iterable<Object[]> parameters() {
                 anyNullIsNull(
                     true,
                     List.of(
-                        new TestCaseSupplier(
-                            "Substring basic test",
-                            List.of(DataTypes.KEYWORD, DataTypes.INTEGER, DataTypes.INTEGER),
-                            () -> {
-                                int start = between(1, 8);
-                                int length = between(1, 10 - start);
-                                String text = randomAlphaOfLength(10);
-                                return new TestCaseSupplier.TestCase(
-                                    List.of(
-                                        new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.KEYWORD, "str"),
-                                        new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                                        new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "end")
-                                    ),
-                                    "SubstringEvaluator[str=Attribute[channel=0], start=Attribute[channel=1], length=Attribute[channel=2]]",
-                                    DataTypes.KEYWORD,
-                                    equalTo(new BytesRef(text.substring(start - 1, start + length - 1)))
-                                );
-                            }
-                        ),
+                        new TestCaseSupplier("Substring basic test", List.of(DataType.KEYWORD, DataType.INTEGER, DataType.INTEGER), () -> {
+                            int start = between(1, 8);
+                            int length = between(1, 10 - start);
+                            String text = randomAlphaOfLength(10);
+                            return new TestCaseSupplier.TestCase(
+                                List.of(
+                                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "end")
+                                ),
+                                "SubstringEvaluator[str=Attribute[channel=0], start=Attribute[channel=1], length=Attribute[channel=2]]",
+                                DataType.KEYWORD,
+                                equalTo(new BytesRef(text.substring(start - 1, start + length - 1)))
+                            );
+                        }),
                         new TestCaseSupplier(
                             "Substring basic test with text input",
-                            List.of(DataTypes.TEXT, DataTypes.INTEGER, DataTypes.INTEGER),
+                            List.of(DataType.TEXT, DataType.INTEGER, DataType.INTEGER),
                             () -> {
                                 int start = between(1, 8);
                                 int length = between(1, 10 - start);
                                 String text = randomAlphaOfLength(10);
                                 return new TestCaseSupplier.TestCase(
                                     List.of(
-                                        new TestCaseSupplier.TypedData(new BytesRef(text), DataTypes.TEXT, "str"),
-                                        new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                                        new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "end")
+                                        new TestCaseSupplier.TypedData(new BytesRef(text), DataType.TEXT, "str"),
+                                        new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                                        new TestCaseSupplier.TypedData(length, DataType.INTEGER, "end")
                                     ),
                                     "SubstringEvaluator[str=Attribute[channel=0], start=Attribute[channel=1], length=Attribute[channel=2]]",
-                                    DataTypes.KEYWORD,
+                                    DataType.KEYWORD,
                                     equalTo(new BytesRef(text.substring(start - 1, start + length - 1)))
                                 );
                             }
                         ),
-                        new TestCaseSupplier(
-                            "Substring empty string",
-                            List.of(DataTypes.TEXT, DataTypes.INTEGER, DataTypes.INTEGER),
-                            () -> {
-                                int start = between(1, 8);
-                                int length = between(1, 10 - start);
-                                return new TestCaseSupplier.TestCase(
-                                    List.of(
-                                        new TestCaseSupplier.TypedData(new BytesRef(""), DataTypes.TEXT, "str"),
-                                        new TestCaseSupplier.TypedData(start, DataTypes.INTEGER, "start"),
-                                        new TestCaseSupplier.TypedData(length, DataTypes.INTEGER, "end")
-                                    ),
-                                    "SubstringEvaluator[str=Attribute[channel=0], start=Attribute[channel=1], length=Attribute[channel=2]]",
-                                    DataTypes.KEYWORD,
-                                    equalTo(new BytesRef(""))
-                                );
-                            }
-                        )
+                        new TestCaseSupplier("Substring empty string", List.of(DataType.TEXT, DataType.INTEGER, DataType.INTEGER), () -> {
+                            int start = between(1, 8);
+                            int length = between(1, 10 - start);
+                            return new TestCaseSupplier.TestCase(
+                                List.of(
+                                    new TestCaseSupplier.TypedData(new BytesRef(""), DataType.TEXT, "str"),
+                                    new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"),
+                                    new TestCaseSupplier.TypedData(length, DataType.INTEGER, "end")
+                                ),
+                                "SubstringEvaluator[str=Attribute[channel=0], start=Attribute[channel=1], length=Attribute[channel=2]]",
+                                DataType.KEYWORD,
+                                equalTo(new BytesRef(""))
+                            );
+                        })
                     )
                 )
             )
@@ -112,7 +104,7 @@ public Matcher<Object> resultsMatcher(List<TestCaseSupplier.TypedData> typedData
 
     public void testNoLengthToString() {
         assertThat(
-            evaluator(new Substring(Source.EMPTY, field("str", DataTypes.KEYWORD), field("start", DataTypes.INTEGER), null)).get(
+            evaluator(new Substring(Source.EMPTY, field("str", DataType.KEYWORD), field("start", DataType.INTEGER), null)).get(
                 driverContext()
             ).toString(),
             equalTo("SubstringNoLengthEvaluator[str=Attribute[channel=0], start=Attribute[channel=1]]")
@@ -187,9 +179,9 @@ private String process(String str, int start, Integer length) {
             EvalOperator.ExpressionEvaluator eval = evaluator(
                 new Substring(
                     Source.EMPTY,
-                    field("str", DataTypes.KEYWORD),
-                    new Literal(Source.EMPTY, start, DataTypes.INTEGER),
-                    length == null ? null : new Literal(Source.EMPTY, length, DataTypes.INTEGER)
+                    field("str", DataType.KEYWORD),
+                    new Literal(Source.EMPTY, start, DataType.INTEGER),
+                    length == null ? null : new Literal(Source.EMPTY, length, DataType.INTEGER)
                 )
             ).get(driverContext());
             Block block = eval.eval(row(List.of(new BytesRef(str))))
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
index cdc8631988638..99b2b38aa8611 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.DateUtils;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
@@ -42,10 +41,10 @@ public ToLowerTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCa
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        suppliers.add(supplier("keyword ascii", DataTypes.KEYWORD, () -> randomAlphaOfLengthBetween(1, 10)));
-        suppliers.add(supplier("keyword unicode", DataTypes.KEYWORD, () -> randomUnicodeOfLengthBetween(1, 10)));
-        suppliers.add(supplier("text ascii", DataTypes.TEXT, () -> randomAlphaOfLengthBetween(1, 10)));
-        suppliers.add(supplier("text unicode", DataTypes.TEXT, () -> randomUnicodeOfLengthBetween(1, 10)));
+        suppliers.add(supplier("keyword ascii", DataType.KEYWORD, () -> randomAlphaOfLengthBetween(1, 10)));
+        suppliers.add(supplier("keyword unicode", DataType.KEYWORD, () -> randomUnicodeOfLengthBetween(1, 10)));
+        suppliers.add(supplier("text ascii", DataType.TEXT, () -> randomAlphaOfLengthBetween(1, 10)));
+        suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10)));
 
         // add null as parameter
         return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(false, suppliers)));
@@ -54,7 +53,7 @@ public static Iterable<Object[]> parameters() {
     public void testRandomLocale() {
         String testString = randomAlphaOfLength(10);
         EsqlConfiguration cfg = randomLocaleConfig();
-        ToLower func = new ToLower(Source.EMPTY, new Literal(Source.EMPTY, testString, DataTypes.KEYWORD), cfg);
+        ToLower func = new ToLower(Source.EMPTY, new Literal(Source.EMPTY, testString, DataType.KEYWORD), cfg);
         assertThat(BytesRefs.toBytesRef(testString.toLowerCase(cfg.locale())), equalTo(func.fold()));
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
index b4ed8e9fbe717..7b8e6abcdb3db 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.DateUtils;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase;
@@ -42,10 +41,10 @@ public ToUpperTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCa
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
 
-        suppliers.add(supplier("keyword ascii", DataTypes.KEYWORD, () -> randomAlphaOfLengthBetween(1, 10)));
-        suppliers.add(supplier("keyword unicode", DataTypes.KEYWORD, () -> randomUnicodeOfLengthBetween(1, 10)));
-        suppliers.add(supplier("text ascii", DataTypes.TEXT, () -> randomAlphaOfLengthBetween(1, 10)));
-        suppliers.add(supplier("text unicode", DataTypes.TEXT, () -> randomUnicodeOfLengthBetween(1, 10)));
+        suppliers.add(supplier("keyword ascii", DataType.KEYWORD, () -> randomAlphaOfLengthBetween(1, 10)));
+        suppliers.add(supplier("keyword unicode", DataType.KEYWORD, () -> randomUnicodeOfLengthBetween(1, 10)));
+        suppliers.add(supplier("text ascii", DataType.TEXT, () -> randomAlphaOfLengthBetween(1, 10)));
+        suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10)));
 
         // add null as parameter
         return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(false, suppliers)));
@@ -54,7 +53,7 @@ public static Iterable<Object[]> parameters() {
     public void testRandomLocale() {
         String testString = randomAlphaOfLength(10);
         EsqlConfiguration cfg = randomLocaleConfig();
-        ToUpper func = new ToUpper(Source.EMPTY, new Literal(Source.EMPTY, testString, DataTypes.KEYWORD), cfg);
+        ToUpper func = new ToUpper(Source.EMPTY, new Literal(Source.EMPTY, testString, DataType.KEYWORD), cfg);
         assertThat(BytesRefs.toBytesRef(testString.toUpperCase(cfg.locale())), equalTo(func.fold()));
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java
index 1e98f10d3c7fc..3aee4a92e9570 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.FunctionName;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -55,7 +54,7 @@ public static Iterable<Object[]> parameters() {
     }
 
     private static void addCases(List<TestCaseSupplier> suppliers) {
-        for (DataType type : new DataType[] { DataTypes.KEYWORD, DataTypes.TEXT }) {
+        for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT }) {
             suppliers.add(new TestCaseSupplier(" with " + type.esType(), List.of(type, type), () -> {
                 BytesRef str = new BytesRef(randomAlphaOfLength(5));
                 String patternString = randomAlphaOfLength(2);
@@ -67,7 +66,7 @@ private static void addCases(List<TestCaseSupplier> suppliers) {
                         new TestCaseSupplier.TypedData(pattern, type, "pattern").forceLiteral()
                     ),
                     startsWith("AutomataMatchEvaluator[input=Attribute[channel=0], pattern=digraph Automaton {\n"),
-                    DataTypes.BOOLEAN,
+                    DataType.BOOLEAN,
                     equalTo(match)
                 );
             }));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java
index b57183e9d56c2..7e803ea2f84a0 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.tree.Location;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -26,8 +25,8 @@
 import java.util.Locale;
 
 import static org.elasticsearch.compute.data.BlockUtils.toJavaObject;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isNull;
 import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.commonType;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isNull;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isRepresentable;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -71,7 +70,7 @@ protected Expression build(Source source, List<Expression> args) {
      * @return True if the type combination is supported by the respective function.
      */
     protected boolean supportsTypes(DataType lhsType, DataType rhsType) {
-        if ((lhsType == DataTypes.UNSIGNED_LONG || rhsType == DataTypes.UNSIGNED_LONG) && lhsType != rhsType) {
+        if ((lhsType == DataType.UNSIGNED_LONG || rhsType == DataType.UNSIGNED_LONG) && lhsType != rhsType) {
             // UL can only be operated on together with another UL, so skip non-UL&UL combinations
             return false;
         }
@@ -80,8 +79,8 @@ protected boolean supportsTypes(DataType lhsType, DataType rhsType) {
 
     public final void testApplyToAllTypes() {
         // TODO replace with test cases
-        for (DataType lhsType : DataTypes.types()) {
-            for (DataType rhsType : DataTypes.types()) {
+        for (DataType lhsType : DataType.types()) {
+            for (DataType rhsType : DataType.types()) {
                 if (supportsTypes(lhsType, rhsType) == false) {
                     continue;
                 }
@@ -117,19 +116,19 @@ public final void testApplyToAllTypes() {
     }
 
     public final void testResolveType() {
-        for (DataType lhsType : DataTypes.types()) {
+        for (DataType lhsType : DataType.types()) {
             if (isRepresentable(lhsType) == false) {
                 continue;
             }
             Literal lhs = randomLiteral(lhsType);
-            for (DataType rhsType : DataTypes.types()) {
+            for (DataType rhsType : DataType.types()) {
                 if (isRepresentable(rhsType) == false) {
                     continue;
                 }
                 Literal rhs = randomLiteral(rhsType);
                 BinaryOperator<?, ?, ?, ?> op = build(new Source(Location.EMPTY, lhsType.typeName() + " " + rhsType.typeName()), lhs, rhs);
 
-                if (lhsType == DataTypes.UNSIGNED_LONG || rhsType == DataTypes.UNSIGNED_LONG) {
+                if (lhsType == DataType.UNSIGNED_LONG || rhsType == DataType.UNSIGNED_LONG) {
                     validateUnsignedLongType(op, lhsType, rhsType);
                     continue;
                 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/BreakerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/BreakerTests.java
index bd85fc3136467..b5bea7d858187 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/BreakerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/BreakerTests.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div;
 import org.junit.After;
@@ -41,8 +41,8 @@ public static Iterable<Object[]> parameters() {
 
         Expression expression = new Div(
             Source.synthetic("[1] / (long) 2"),
-            AbstractFunctionTestCase.field("f", DataTypes.LONG),
-            new Literal(Source.EMPTY, 2, DataTypes.INTEGER)
+            AbstractFunctionTestCase.field("f", DataType.LONG),
+            new Literal(Source.EMPTY, 2, DataType.INTEGER)
         );
         for (int b = 0; b < 136; b++) {
             params.add(new Object[] { ByteSizeValue.ofBytes(b), expression });
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java
index 0fa5b6121a6af..141fc24e73e18 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticTestCase.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.AbstractBinaryOperatorTestCase;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -32,7 +31,7 @@ protected Matcher<Object> resultMatcher(List<Object> data, DataType dataType) {
             return equalTo(expectedValue(lhs.doubleValue(), rhs.doubleValue()));
         }
         if (lhs instanceof Long || rhs instanceof Long) {
-            if (dataType == DataTypes.UNSIGNED_LONG) {
+            if (dataType == DataType.UNSIGNED_LONG) {
                 return equalTo(expectedUnsignedLongValue(lhs.longValue(), rhs.longValue()));
             }
             return equalTo(expectedValue(lhs.longValue(), rhs.longValue()));
@@ -47,16 +46,16 @@ protected Matcher<Object> resultMatcher(List<Object> data, DataType dataType) {
     protected Matcher<Object> resultsMatcher(List<TestCaseSupplier.TypedData> typedData) {
         Number lhs = (Number) typedData.get(0).data();
         Number rhs = (Number) typedData.get(1).data();
-        if (typedData.stream().anyMatch(t -> t.type().equals(DataTypes.DOUBLE))) {
+        if (typedData.stream().anyMatch(t -> t.type().equals(DataType.DOUBLE))) {
             return equalTo(expectedValue(lhs.doubleValue(), rhs.doubleValue()));
         }
-        if (typedData.stream().anyMatch(t -> t.type().equals(DataTypes.UNSIGNED_LONG))) {
+        if (typedData.stream().anyMatch(t -> t.type().equals(DataType.UNSIGNED_LONG))) {
             return equalTo(expectedUnsignedLongValue(lhs.longValue(), rhs.longValue()));
         }
-        if (typedData.stream().anyMatch(t -> t.type().equals(DataTypes.LONG))) {
+        if (typedData.stream().anyMatch(t -> t.type().equals(DataType.LONG))) {
             return equalTo(expectedValue(lhs.longValue(), rhs.longValue()));
         }
-        if (typedData.stream().anyMatch(t -> t.type().equals(DataTypes.INTEGER))) {
+        if (typedData.stream().anyMatch(t -> t.type().equals(DataType.INTEGER))) {
             return equalTo(expectedValue(lhs.intValue(), rhs.intValue()));
         }
         throw new UnsupportedOperationException();
@@ -77,7 +76,7 @@ protected boolean supportsType(DataType type) {
 
     @Override
     protected void validateType(BinaryOperator<?, ?, ?, ?> op, DataType lhsType, DataType rhsType) {
-        if (DataTypes.isNullOrNumeric(lhsType) && DataTypes.isNullOrNumeric(rhsType)) {
+        if (DataType.isNullOrNumeric(lhsType) && DataType.isNullOrNumeric(rhsType)) {
             assertTrue(op.toString(), op.typeResolved().resolved());
             assertThat(op.toString(), op.dataType(), equalTo(expectedType(lhsType, rhsType)));
             return;
@@ -102,22 +101,22 @@ protected void validateType(BinaryOperator<?, ?, ?, ?> op, DataType lhsType, Dat
     }
 
     protected DataType expectedType(DataType lhsType, DataType rhsType) {
-        if (lhsType == DataTypes.DOUBLE || rhsType == DataTypes.DOUBLE) {
-            return DataTypes.DOUBLE;
+        if (lhsType == DataType.DOUBLE || rhsType == DataType.DOUBLE) {
+            return DataType.DOUBLE;
         }
-        if (lhsType == DataTypes.UNSIGNED_LONG || rhsType == DataTypes.UNSIGNED_LONG) {
-            assertThat(lhsType, is(DataTypes.UNSIGNED_LONG));
-            assertThat(rhsType, is(DataTypes.UNSIGNED_LONG));
-            return DataTypes.UNSIGNED_LONG;
+        if (lhsType == DataType.UNSIGNED_LONG || rhsType == DataType.UNSIGNED_LONG) {
+            assertThat(lhsType, is(DataType.UNSIGNED_LONG));
+            assertThat(rhsType, is(DataType.UNSIGNED_LONG));
+            return DataType.UNSIGNED_LONG;
         }
-        if (lhsType == DataTypes.LONG || rhsType == DataTypes.LONG) {
-            return DataTypes.LONG;
+        if (lhsType == DataType.LONG || rhsType == DataType.LONG) {
+            return DataType.LONG;
         }
-        if (lhsType == DataTypes.INTEGER || rhsType == DataTypes.INTEGER) {
-            return DataTypes.INTEGER;
+        if (lhsType == DataType.INTEGER || rhsType == DataType.INTEGER) {
+            return DataType.INTEGER;
         }
-        if (lhsType == DataTypes.NULL || rhsType == DataTypes.NULL) {
-            return DataTypes.NULL;
+        if (lhsType == DataType.NULL || rhsType == DataType.NULL) {
+            return DataType.NULL;
         }
         throw new UnsupportedOperationException();
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java
index 5c137c1914d7b..8a27a289bb77f 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractDateTimeArithmeticTestCase.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.hamcrest.Matcher;
 
@@ -19,8 +18,8 @@
 import java.util.List;
 import java.util.Locale;
 
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isDateTime;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.isNull;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isDateTime;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isNull;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isNullOrTemporalAmount;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
 import static org.hamcrest.Matchers.equalTo;
@@ -108,7 +107,7 @@ private void assertTypeResolution(String failureMessage, BinaryOperator<?, ?, ?,
     @Override
     protected DataType expectedType(DataType lhsType, DataType rhsType) {
         if (isDateTime(lhsType) || isDateTime(rhsType)) {
-            return DataTypes.DATETIME;
+            return DataType.DATETIME;
         } else if (isNullOrTemporalAmount(lhsType) || isNullOrTemporalAmount(rhsType)) {
             if (isNull(lhsType)) {
                 return rhsType;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
index 4d6e6015073a9..26a5d58b33900 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -83,7 +82,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> (((BigInteger) l).add((BigInteger) r)),
-                DataTypes.UNSIGNED_LONG,
+                DataType.UNSIGNED_LONG,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE), true),
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE), true),
                 List.of(),
@@ -96,7 +95,7 @@ public static Iterable<Object[]> parameters() {
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 (lhs, rhs) -> ((Period) lhs).plus((Period) rhs),
-                DataTypes.DATE_PERIOD,
+                DataType.DATE_PERIOD,
                 TestCaseSupplier.datePeriodCases(),
                 TestCaseSupplier.datePeriodCases(),
                 startsWith("LiteralsEvaluator[lit="),  // lhs and rhs have to be literals, so we fold into a literal
@@ -107,7 +106,7 @@ public static Iterable<Object[]> parameters() {
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 (lhs, rhs) -> ((Duration) lhs).plus((Duration) rhs),
-                DataTypes.TIME_DURATION,
+                DataType.TIME_DURATION,
                 TestCaseSupplier.timeDurationCases(),
                 TestCaseSupplier.timeDurationCases(),
                 startsWith("LiteralsEvaluator[lit="), // lhs and rhs have to be literals, so we fold into a literal
@@ -137,7 +136,7 @@ public static Iterable<Object[]> parameters() {
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 result,
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 TestCaseSupplier.dateCases(),
                 TestCaseSupplier.datePeriodCases(),
                 startsWith("AddDatetimesEvaluator[datetime=Attribute[channel=0], temporalAmount="),
@@ -148,7 +147,7 @@ public static Iterable<Object[]> parameters() {
         suppliers.addAll(
             TestCaseSupplier.forBinaryNotCasting(
                 result,
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 TestCaseSupplier.dateCases(),
                 TestCaseSupplier.timeDurationCases(),
                 startsWith("AddDatetimesEvaluator[datetime=Attribute[channel=0], temporalAmount="),
@@ -159,22 +158,22 @@ public static Iterable<Object[]> parameters() {
         suppliers.addAll(TestCaseSupplier.dateCases().stream().<TestCaseSupplier>mapMulti((tds, consumer) -> {
             consumer.accept(
                 new TestCaseSupplier(
-                    List.of(DataTypes.DATETIME, DataTypes.NULL),
+                    List.of(DataType.DATETIME, DataType.NULL),
                     () -> new TestCaseSupplier.TestCase(
                         List.of(tds.get(), TestCaseSupplier.TypedData.NULL),
                         "LiteralsEvaluator[lit=null]",
-                        DataTypes.DATETIME,
+                        DataType.DATETIME,
                         nullValue()
                     )
                 )
             );
             consumer.accept(
                 new TestCaseSupplier(
-                    List.of(DataTypes.NULL, DataTypes.DATETIME),
+                    List.of(DataType.NULL, DataType.DATETIME),
                     () -> new TestCaseSupplier.TestCase(
                         List.of(TestCaseSupplier.TypedData.NULL, tds.get()),
                         "LiteralsEvaluator[lit=null]",
-                        DataTypes.DATETIME,
+                        DataType.DATETIME,
                         nullValue()
                     )
                 )
@@ -198,11 +197,11 @@ public static Iterable<Object[]> parameters() {
             int lhs2 = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataTypes.INTEGER, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, DataTypes.INTEGER, "rhs")
+                    new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataType.INTEGER, "lhs"),
+                    new TestCaseSupplier.TypedData(rhs, DataType.INTEGER, "rhs")
                 ),
                 "AddIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 is(nullValue())
             ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
                 .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value");
@@ -210,7 +209,7 @@ public static Iterable<Object[]> parameters() {
         // exact math arithmetic exceptions
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 () -> randomIntBetween(1, Integer.MAX_VALUE),
                 () -> Integer.MAX_VALUE,
                 "AddIntsEvaluator"
@@ -218,7 +217,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 () -> randomIntBetween(Integer.MIN_VALUE, -1),
                 () -> Integer.MIN_VALUE,
                 "AddIntsEvaluator"
@@ -226,7 +225,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.LONG,
+                DataType.LONG,
                 () -> randomLongBetween(1L, Long.MAX_VALUE),
                 () -> Long.MAX_VALUE,
                 "AddLongsEvaluator"
@@ -234,7 +233,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.LONG,
+                DataType.LONG,
                 () -> randomLongBetween(Long.MIN_VALUE, -1L),
                 () -> Long.MIN_VALUE,
                 "AddLongsEvaluator"
@@ -242,7 +241,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.UNSIGNED_LONG,
+                DataType.UNSIGNED_LONG,
                 () -> asLongUnsigned(randomBigInteger()),
                 () -> asLongUnsigned(UNSIGNED_LONG_MAX),
                 "AddUnsignedLongsEvaluator"
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java
index 173ba55cefdac..a50d44822a4e3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -61,7 +60,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (lhs, rhs) -> {
-                    if (lhs.type() != DataTypes.DOUBLE || rhs.type() != DataTypes.DOUBLE) {
+                    if (lhs.type() != DataType.DOUBLE || rhs.type() != DataType.DOUBLE) {
                         return List.of();
                     }
                     double v = ((Double) lhs.getValue()) / ((Double) rhs.getValue());
@@ -82,7 +81,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> (((BigInteger) l).divide((BigInteger) r)),
-                DataTypes.UNSIGNED_LONG,
+                DataType.UNSIGNED_LONG,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE), true),
                 TestCaseSupplier.ulongCases(BigInteger.ONE, BigInteger.valueOf(Long.MAX_VALUE), true),
                 List.of(),
@@ -113,7 +112,7 @@ public static Iterable<Object[]> parameters() {
                 "DivDoublesEvaluator"
             )
         );
-        List<DataType> numericTypes = List.of(DataTypes.INTEGER, DataTypes.LONG, DataTypes.DOUBLE);
+        List<DataType> numericTypes = List.of(DataType.INTEGER, DataType.LONG, DataType.DOUBLE);
 
         for (DataType lhsType : numericTypes) {
             for (DataType rhsType : numericTypes) {
@@ -153,7 +152,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> null,
-                DataTypes.UNSIGNED_LONG,
+                DataType.UNSIGNED_LONG,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE), true),
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.ZERO, true),
                 List.of(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java
index df79ee203a3f8..ce67f6453362b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.hamcrest.Matcher;
@@ -69,7 +68,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> (((BigInteger) l).mod((BigInteger) r)),
-                DataTypes.UNSIGNED_LONG,
+                DataType.UNSIGNED_LONG,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE), true),
                 TestCaseSupplier.ulongCases(BigInteger.ONE, BigInteger.valueOf(Long.MAX_VALUE), true),
                 List.of(),
@@ -100,7 +99,7 @@ public static Iterable<Object[]> parameters() {
                 "ModDoublesEvaluator"
             )
         );
-        List<DataType> numericTypes = List.of(DataTypes.INTEGER, DataTypes.LONG, DataTypes.DOUBLE);
+        List<DataType> numericTypes = List.of(DataType.INTEGER, DataType.LONG, DataType.DOUBLE);
 
         for (DataType lhsType : numericTypes) {
             for (DataType rhsType : numericTypes) {
@@ -140,7 +139,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> null,
-                DataTypes.UNSIGNED_LONG,
+                DataType.UNSIGNED_LONG,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE), true),
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.ZERO, true),
                 List.of(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java
index faae48920f351..8b4dfa88415be 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -56,22 +56,22 @@ public static Iterable<Object[]> parameters() {
             )
         );
 
-        suppliers.add(new TestCaseSupplier("Double * Double", List.of(DataTypes.DOUBLE, DataTypes.DOUBLE), () -> {
+        suppliers.add(new TestCaseSupplier("Double * Double", List.of(DataType.DOUBLE, DataType.DOUBLE), () -> {
             double rhs = randomDouble();
             double lhs = randomDouble();
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(lhs, DataTypes.DOUBLE, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, DataTypes.DOUBLE, "rhs")
+                    new TestCaseSupplier.TypedData(lhs, DataType.DOUBLE, "lhs"),
+                    new TestCaseSupplier.TypedData(rhs, DataType.DOUBLE, "rhs")
                 ),
                 "MulDoublesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 equalTo(lhs * rhs)
             );
         }));
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 () -> randomBoolean() ? Integer.MIN_VALUE : Integer.MAX_VALUE,
                 () -> randomIntBetween(2, Integer.MAX_VALUE),
                 "MulIntsEvaluator"
@@ -79,7 +79,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.LONG,
+                DataType.LONG,
                 () -> randomBoolean() ? Long.MIN_VALUE : Long.MAX_VALUE,
                 () -> randomLongBetween(2L, Long.MAX_VALUE),
                 "MulLongsEvaluator"
@@ -87,7 +87,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.UNSIGNED_LONG,
+                DataType.UNSIGNED_LONG,
                 () -> asLongUnsigned(UNSIGNED_LONG_MAX),
                 () -> asLongUnsigned(randomLongBetween(-Long.MAX_VALUE, Long.MAX_VALUE)),
                 "MulUnsignedLongsEvaluator"
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
index 784599678d65b..c2a9766c23cbe 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -41,7 +40,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "NegIntsEvaluator[v=Attribute[channel=0]]",
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             Math::negateExact,
             Integer.MIN_VALUE + 1,
             Integer.MAX_VALUE,
@@ -51,7 +50,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryInt(
             suppliers,
             "NegIntsEvaluator[v=Attribute[channel=0]]",
-            DataTypes.INTEGER,
+            DataType.INTEGER,
             z -> null,
             Integer.MIN_VALUE,
             Integer.MIN_VALUE,
@@ -63,7 +62,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             "NegLongsEvaluator[v=Attribute[channel=0]]",
-            DataTypes.LONG,
+            DataType.LONG,
             Math::negateExact,
             Long.MIN_VALUE + 1,
             Long.MAX_VALUE,
@@ -73,7 +72,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryLong(
             suppliers,
             "NegLongsEvaluator[v=Attribute[channel=0]]",
-            DataTypes.LONG,
+            DataType.LONG,
             z -> null,
             Long.MIN_VALUE,
             Long.MIN_VALUE,
@@ -85,7 +84,7 @@ public static Iterable<Object[]> parameters() {
         TestCaseSupplier.forUnaryDouble(
             suppliers,
             "NegDoublesEvaluator[v=Attribute[channel=0]]",
-            DataTypes.DOUBLE,
+            DataType.DOUBLE,
             // TODO: Probably we don't want to allow negative zeros
             d -> -d,
             Double.NEGATIVE_INFINITY,
@@ -94,20 +93,20 @@ public static Iterable<Object[]> parameters() {
         );
 
         // TODO: Wire up edge case generation functions for these
-        suppliers.addAll(List.of(new TestCaseSupplier("Duration", List.of(DataTypes.TIME_DURATION), () -> {
-            Duration arg = (Duration) randomLiteral(DataTypes.TIME_DURATION).value();
+        suppliers.addAll(List.of(new TestCaseSupplier("Duration", List.of(DataType.TIME_DURATION), () -> {
+            Duration arg = (Duration) randomLiteral(DataType.TIME_DURATION).value();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(arg, DataTypes.TIME_DURATION, "arg")),
+                List.of(new TestCaseSupplier.TypedData(arg, DataType.TIME_DURATION, "arg")),
                 "No evaluator since this expression is only folded",
-                DataTypes.TIME_DURATION,
+                DataType.TIME_DURATION,
                 equalTo(arg.negated())
             );
-        }), new TestCaseSupplier("Period", List.of(DataTypes.DATE_PERIOD), () -> {
-            Period arg = (Period) randomLiteral(DataTypes.DATE_PERIOD).value();
+        }), new TestCaseSupplier("Period", List.of(DataType.DATE_PERIOD), () -> {
+            Period arg = (Period) randomLiteral(DataType.DATE_PERIOD).value();
             return new TestCaseSupplier.TestCase(
-                List.of(new TestCaseSupplier.TypedData(arg, DataTypes.DATE_PERIOD, "arg")),
+                List.of(new TestCaseSupplier.TypedData(arg, DataType.DATE_PERIOD, "arg")),
                 "No evaluator since this expression is only folded",
-                DataTypes.DATE_PERIOD,
+                DataType.DATE_PERIOD,
                 equalTo(arg.negated())
             );
         })));
@@ -123,7 +122,7 @@ public void testEdgeCases() {
         // Run the assertions for the current test cases type only to avoid running the same assertions multiple times.
         // TODO: These remaining cases should get rolled into generation functions for periods and durations
         DataType testCaseType = testCase.getData().get(0).type();
-        if (testCaseType == DataTypes.DATE_PERIOD) {
+        if (testCaseType == DataType.DATE_PERIOD) {
             Period maxPeriod = Period.of(Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE);
             Period negatedMaxPeriod = Period.of(-Integer.MAX_VALUE, -Integer.MAX_VALUE, -Integer.MAX_VALUE);
             assertEquals(negatedMaxPeriod, process(maxPeriod));
@@ -135,7 +134,7 @@ public void testEdgeCases() {
                 () -> process(minPeriod)
             );
             assertEquals(e.getMessage(), "arithmetic exception in expression []: [integer overflow]");
-        } else if (testCaseType == DataTypes.TIME_DURATION) {
+        } else if (testCaseType == DataType.TIME_DURATION) {
             Duration maxDuration = Duration.ofSeconds(Long.MAX_VALUE, 0);
             Duration negatedMaxDuration = Duration.ofSeconds(-Long.MAX_VALUE, 0);
             assertEquals(negatedMaxDuration, process(maxDuration));
@@ -167,19 +166,19 @@ private Object process(Object val) {
 
     private static DataType typeOf(Object val) {
         if (val instanceof Integer) {
-            return DataTypes.INTEGER;
+            return DataType.INTEGER;
         }
         if (val instanceof Long) {
-            return DataTypes.LONG;
+            return DataType.LONG;
         }
         if (val instanceof Double) {
-            return DataTypes.DOUBLE;
+            return DataType.DOUBLE;
         }
         if (val instanceof Duration) {
-            return DataTypes.TIME_DURATION;
+            return DataType.TIME_DURATION;
         }
         if (val instanceof Period) {
-            return DataTypes.DATE_PERIOD;
+            return DataType.DATE_PERIOD;
         }
         throw new UnsupportedOperationException("unsupported type [" + val.getClass() + "]");
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
index f3c63569d458d..e75ee9333ba54 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -73,62 +73,62 @@ public static Iterable<Object[]> parameters() {
             BigInteger rhsBI = unsignedLongAsBigInteger(rhs);
             return new TestCase(
                 Source.EMPTY,
-                List.of(new TypedData(lhs, DataTypes.UNSIGNED_LONG, "lhs"), new TypedData(rhs, DataTypes.UNSIGNED_LONG, "rhs")),
+                List.of(new TypedData(lhs, DataType.UNSIGNED_LONG, "lhs"), new TypedData(rhs, DataType.UNSIGNED_LONG, "rhs")),
                 "SubUnsignedLongsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
                 equalTo(asLongUnsigned(lhsBI.subtract(rhsBI).longValue()))
             );
           }) */
 
         suppliers.add(new TestCaseSupplier("Datetime - Period", () -> {
-            long lhs = (Long) randomLiteral(DataTypes.DATETIME).value();
-            Period rhs = (Period) randomLiteral(DataTypes.DATE_PERIOD).value();
+            long lhs = (Long) randomLiteral(DataType.DATETIME).value();
+            Period rhs = (Period) randomLiteral(DataType.DATE_PERIOD).value();
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(lhs, DataTypes.DATETIME, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, DataTypes.DATE_PERIOD, "rhs")
+                    new TestCaseSupplier.TypedData(lhs, DataType.DATETIME, "lhs"),
+                    new TestCaseSupplier.TypedData(rhs, DataType.DATE_PERIOD, "rhs")
                 ),
                 "SubDatetimesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 equalTo(asMillis(asDateTime(lhs).minus(rhs)))
             );
         }));
         suppliers.add(new TestCaseSupplier("Period - Period", () -> {
-            Period lhs = (Period) randomLiteral(DataTypes.DATE_PERIOD).value();
-            Period rhs = (Period) randomLiteral(DataTypes.DATE_PERIOD).value();
+            Period lhs = (Period) randomLiteral(DataType.DATE_PERIOD).value();
+            Period rhs = (Period) randomLiteral(DataType.DATE_PERIOD).value();
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(lhs, DataTypes.DATE_PERIOD, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, DataTypes.DATE_PERIOD, "rhs")
+                    new TestCaseSupplier.TypedData(lhs, DataType.DATE_PERIOD, "lhs"),
+                    new TestCaseSupplier.TypedData(rhs, DataType.DATE_PERIOD, "rhs")
                 ),
                 "Only folding possible, so there's no evaluator",
-                DataTypes.DATE_PERIOD,
+                DataType.DATE_PERIOD,
                 equalTo(lhs.minus(rhs))
             );
         }));
         suppliers.add(new TestCaseSupplier("Datetime - Duration", () -> {
-            long lhs = (Long) randomLiteral(DataTypes.DATETIME).value();
-            Duration rhs = (Duration) randomLiteral(DataTypes.TIME_DURATION).value();
+            long lhs = (Long) randomLiteral(DataType.DATETIME).value();
+            Duration rhs = (Duration) randomLiteral(DataType.TIME_DURATION).value();
             TestCaseSupplier.TestCase testCase = new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(lhs, DataTypes.DATETIME, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, DataTypes.TIME_DURATION, "rhs")
+                    new TestCaseSupplier.TypedData(lhs, DataType.DATETIME, "lhs"),
+                    new TestCaseSupplier.TypedData(rhs, DataType.TIME_DURATION, "rhs")
                 ),
                 "SubDatetimesEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 equalTo(asMillis(asDateTime(lhs).minus(rhs)))
             );
             return testCase;
         }));
         suppliers.add(new TestCaseSupplier("Duration - Duration", () -> {
-            Duration lhs = (Duration) randomLiteral(DataTypes.TIME_DURATION).value();
-            Duration rhs = (Duration) randomLiteral(DataTypes.TIME_DURATION).value();
+            Duration lhs = (Duration) randomLiteral(DataType.TIME_DURATION).value();
+            Duration rhs = (Duration) randomLiteral(DataType.TIME_DURATION).value();
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(lhs, DataTypes.TIME_DURATION, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, DataTypes.TIME_DURATION, "rhs")
+                    new TestCaseSupplier.TypedData(lhs, DataType.TIME_DURATION, "lhs"),
+                    new TestCaseSupplier.TypedData(rhs, DataType.TIME_DURATION, "rhs")
                 ),
                 "Only folding possible, so there's no evaluator",
-                DataTypes.TIME_DURATION,
+                DataType.TIME_DURATION,
                 equalTo(lhs.minus(rhs))
             );
         }));
@@ -139,11 +139,11 @@ public static Iterable<Object[]> parameters() {
             int lhs2 = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataTypes.INTEGER, "lhs"),
-                    new TestCaseSupplier.TypedData(rhs, DataTypes.INTEGER, "rhs")
+                    new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataType.INTEGER, "lhs"),
+                    new TestCaseSupplier.TypedData(rhs, DataType.INTEGER, "rhs")
                 ),
                 "SubIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 is(nullValue())
             ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
                 .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value");
@@ -151,7 +151,7 @@ public static Iterable<Object[]> parameters() {
         // exact math arithmetic exceptions
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 () -> Integer.MIN_VALUE,
                 () -> randomIntBetween(1, Integer.MAX_VALUE),
                 "SubIntsEvaluator"
@@ -159,7 +159,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 () -> randomIntBetween(Integer.MIN_VALUE, -2),
                 () -> Integer.MAX_VALUE,
                 "SubIntsEvaluator"
@@ -167,7 +167,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.LONG,
+                DataType.LONG,
                 () -> Long.MIN_VALUE,
                 () -> randomLongBetween(1L, Long.MAX_VALUE),
                 "SubLongsEvaluator"
@@ -175,7 +175,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.LONG,
+                DataType.LONG,
                 () -> randomLongBetween(Long.MIN_VALUE, -2L),
                 () -> Long.MAX_VALUE,
                 "SubLongsEvaluator"
@@ -183,7 +183,7 @@ public static Iterable<Object[]> parameters() {
         );
         suppliers.add(
             arithmeticExceptionOverflowCase(
-                DataTypes.UNSIGNED_LONG,
+                DataType.UNSIGNED_LONG,
                 () -> ZERO_AS_UNSIGNED_LONG,
                 () -> randomLongBetween(-Long.MAX_VALUE, Long.MAX_VALUE),
                 "SubUnsignedLongsEvaluator"
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java
index 77a80a4b60ce6..3817bbe9cc74c 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -69,7 +69,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 Object::equals,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true),
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true),
                 List.of(),
@@ -82,7 +82,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 Object::equals,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.booleanCases(),
                 TestCaseSupplier.booleanCases(),
                 List.of(),
@@ -95,7 +95,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 Object::equals,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ipCases(),
                 TestCaseSupplier.ipCases(),
                 List.of(),
@@ -108,7 +108,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 Object::equals,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.versionCases(""),
                 TestCaseSupplier.versionCases(""),
                 List.of(),
@@ -123,7 +123,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 Object::equals,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.dateCases(),
                 TestCaseSupplier.dateCases(),
                 List.of(),
@@ -136,7 +136,7 @@ public static Iterable<Object[]> parameters() {
                 Object::equals,
                 (lhsType, rhsType) -> "EqualsKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
                 List.of(),
-                DataTypes.BOOLEAN
+                DataType.BOOLEAN
             )
         );
 
@@ -146,7 +146,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 Object::equals,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.geoPointCases(),
                 TestCaseSupplier.geoPointCases(),
                 List.of(),
@@ -160,7 +160,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 Object::equals,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.geoShapeCases(),
                 TestCaseSupplier.geoShapeCases(),
                 List.of(),
@@ -173,7 +173,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 Object::equals,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.cartesianPointCases(),
                 TestCaseSupplier.cartesianPointCases(),
                 List.of(),
@@ -187,7 +187,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 Object::equals,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.cartesianShapeCases(),
                 TestCaseSupplier.cartesianShapeCases(),
                 List.of(),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java
index 99f4b4ff82d96..f25638b482817 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -70,7 +70,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BigInteger) l).compareTo((BigInteger) r) >= 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true),
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true),
                 List.of(),
@@ -84,7 +84,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) >= 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ipCases(),
                 TestCaseSupplier.ipCases(),
                 List.of(),
@@ -98,7 +98,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) >= 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.versionCases(""),
                 TestCaseSupplier.versionCases(""),
                 List.of(),
@@ -113,7 +113,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((Number) l).longValue() >= ((Number) r).longValue(),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.dateCases(),
                 TestCaseSupplier.dateCases(),
                 List.of(),
@@ -126,7 +126,7 @@ public static Iterable<Object[]> parameters() {
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) >= 0,
                 (lhsType, rhsType) -> "GreaterThanOrEqualKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
                 List.of(),
-                DataTypes.BOOLEAN
+                DataType.BOOLEAN
             )
         );
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java
index f0fe60c458c78..0735e0dfd64f2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -70,7 +70,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BigInteger) l).compareTo((BigInteger) r) > 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true),
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true),
                 List.of(),
@@ -84,7 +84,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) > 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ipCases(),
                 TestCaseSupplier.ipCases(),
                 List.of(),
@@ -98,7 +98,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) > 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.versionCases(""),
                 TestCaseSupplier.versionCases(""),
                 List.of(),
@@ -113,7 +113,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((Number) l).longValue() > ((Number) r).longValue(),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.dateCases(),
                 TestCaseSupplier.dateCases(),
                 List.of(),
@@ -126,7 +126,7 @@ public static Iterable<Object[]> parameters() {
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) > 0,
                 (lhsType, rhsType) -> "GreaterThanKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
                 List.of(),
-                DataTypes.BOOLEAN
+                DataType.BOOLEAN
             )
         );
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java
index 56758d43a83e6..4a802dfcaf975 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -70,7 +70,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BigInteger) l).compareTo((BigInteger) r) <= 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true),
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true),
                 List.of(),
@@ -84,7 +84,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) <= 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ipCases(),
                 TestCaseSupplier.ipCases(),
                 List.of(),
@@ -98,7 +98,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) <= 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.versionCases(""),
                 TestCaseSupplier.versionCases(""),
                 List.of(),
@@ -113,7 +113,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((Number) l).longValue() <= ((Number) r).longValue(),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.dateCases(),
                 TestCaseSupplier.dateCases(),
                 List.of(),
@@ -126,7 +126,7 @@ public static Iterable<Object[]> parameters() {
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) <= 0,
                 (lhsType, rhsType) -> "LessThanOrEqualKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
                 List.of(),
-                DataTypes.BOOLEAN
+                DataType.BOOLEAN
             )
         );
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
index fa6f3fbd0926c..6f3f2441c6d00 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
@@ -70,7 +70,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BigInteger) l).compareTo((BigInteger) r) < 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true),
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true),
                 List.of(),
@@ -84,7 +84,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) < 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ipCases(),
                 TestCaseSupplier.ipCases(),
                 List.of(),
@@ -98,7 +98,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) < 0,
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.versionCases(""),
                 TestCaseSupplier.versionCases(""),
                 List.of(),
@@ -113,7 +113,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> ((Number) l).longValue() < ((Number) r).longValue(),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.dateCases(),
                 TestCaseSupplier.dateCases(),
                 List.of(),
@@ -126,7 +126,7 @@ public static Iterable<Object[]> parameters() {
                 (l, r) -> ((BytesRef) l).compareTo((BytesRef) r) < 0,
                 (lhsType, rhsType) -> "LessThanKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
                 List.of(),
-                DataTypes.BOOLEAN
+                DataType.BOOLEAN
             )
         );
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java
index e03a569904dc4..174e2457eb0a5 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java
@@ -12,7 +12,7 @@
 
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -68,7 +68,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> false == l.equals(r),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE), true),
                 TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE), true),
                 List.of(),
@@ -81,7 +81,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> false == l.equals(r),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.booleanCases(),
                 TestCaseSupplier.booleanCases(),
                 List.of(),
@@ -94,7 +94,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> false == l.equals(r),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.ipCases(),
                 TestCaseSupplier.ipCases(),
                 List.of(),
@@ -107,7 +107,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> false == l.equals(r),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.versionCases(""),
                 TestCaseSupplier.versionCases(""),
                 List.of(),
@@ -122,7 +122,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> false == l.equals(r),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.dateCases(),
                 TestCaseSupplier.dateCases(),
                 List.of(),
@@ -134,7 +134,7 @@ public static Iterable<Object[]> parameters() {
                 (l, r) -> false == l.equals(r),
                 (lhsType, rhsType) -> "NotEqualsKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]",
                 List.of(),
-                DataTypes.BOOLEAN
+                DataType.BOOLEAN
             )
         );
         suppliers.addAll(
@@ -143,7 +143,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> false == l.equals(r),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.geoPointCases(),
                 TestCaseSupplier.geoPointCases(),
                 List.of(),
@@ -156,7 +156,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> false == l.equals(r),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.geoShapeCases(),
                 TestCaseSupplier.geoShapeCases(),
                 List.of(),
@@ -169,7 +169,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> false == l.equals(r),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.cartesianPointCases(),
                 TestCaseSupplier.cartesianPointCases(),
                 List.of(),
@@ -182,7 +182,7 @@ public static Iterable<Object[]> parameters() {
                 "lhs",
                 "rhs",
                 (l, r) -> false == l.equals(r),
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 TestCaseSupplier.cartesianShapeCases(),
                 TestCaseSupplier.cartesianShapeCases(),
                 List.of(),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
index 9cb4b6cff3fc0..55c3811f9a870 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
 import org.elasticsearch.xpack.esql.core.type.KeywordEsField;
@@ -205,7 +204,7 @@ public void testWrappedStreamSimple() throws IOException {
         BytesStreamOutput bso = new BytesStreamOutput();
         bso.writeString("hello");
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        var plan = new RowExec(Source.EMPTY, List.of(new Alias(Source.EMPTY, "foo", field("field", DataTypes.LONG))));
+        var plan = new RowExec(Source.EMPTY, List.of(new Alias(Source.EMPTY, "foo", field("field", DataType.LONG))));
         out.writePhysicalPlanNode(plan);
         bso.writeVInt(11_345);
 
@@ -219,7 +218,7 @@ public void testWrappedStreamSimple() throws IOException {
     }
 
     public void testBinComparisonSimple() throws IOException {
-        var orig = new Equals(Source.EMPTY, field("foo", DataTypes.DOUBLE), field("bar", DataTypes.DOUBLE));
+        var orig = new Equals(Source.EMPTY, field("foo", DataType.DOUBLE), field("bar", DataType.DOUBLE));
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         out.writeNamed(EsqlBinaryComparison.class, orig);
@@ -234,7 +233,7 @@ public void testBinComparison() {
     }
 
     public void testAggFunctionSimple() throws IOException {
-        var orig = new Avg(Source.EMPTY, field("foo_val", DataTypes.DOUBLE));
+        var orig = new Avg(Source.EMPTY, field("foo_val", DataType.DOUBLE));
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         out.writeNamed(AggregateFunction.class, orig);
@@ -247,7 +246,7 @@ public void testAggFunction() {
     }
 
     public void testArithmeticOperationSimple() throws IOException {
-        var orig = new Add(Source.EMPTY, field("foo", DataTypes.LONG), field("bar", DataTypes.LONG));
+        var orig = new Add(Source.EMPTY, field("foo", DataType.LONG), field("bar", DataType.LONG));
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         out.writeNamed(ArithmeticOperation.class, orig);
@@ -262,7 +261,7 @@ public void testArithmeticOperation() {
     }
 
     public void testSubStringSimple() throws IOException {
-        var orig = new Substring(Source.EMPTY, field("foo", DataTypes.KEYWORD), new Literal(Source.EMPTY, 1, DataTypes.INTEGER), null);
+        var orig = new Substring(Source.EMPTY, field("foo", DataType.KEYWORD), new Literal(Source.EMPTY, 1, DataType.INTEGER), null);
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         PlanNamedTypes.writeSubstring(out, orig);
@@ -271,7 +270,7 @@ public void testSubStringSimple() throws IOException {
     }
 
     public void testStartsWithSimple() throws IOException {
-        var orig = new StartsWith(Source.EMPTY, field("foo", DataTypes.KEYWORD), new Literal(Source.EMPTY, "fo", DataTypes.KEYWORD));
+        var orig = new StartsWith(Source.EMPTY, field("foo", DataType.KEYWORD), new Literal(Source.EMPTY, "fo", DataType.KEYWORD));
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         PlanNamedTypes.writeStartsWith(out, orig);
@@ -280,7 +279,7 @@ public void testStartsWithSimple() throws IOException {
     }
 
     public void testRoundSimple() throws IOException {
-        var orig = new Round(Source.EMPTY, field("value", DataTypes.DOUBLE), new Literal(Source.EMPTY, 1, DataTypes.INTEGER));
+        var orig = new Round(Source.EMPTY, field("value", DataType.DOUBLE), new Literal(Source.EMPTY, 1, DataType.INTEGER));
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         PlanNamedTypes.writeRound(out, orig);
@@ -289,7 +288,7 @@ public void testRoundSimple() throws IOException {
     }
 
     public void testPowSimple() throws IOException {
-        var orig = new Pow(Source.EMPTY, field("value", DataTypes.DOUBLE), new Literal(Source.EMPTY, 1, DataTypes.INTEGER));
+        var orig = new Pow(Source.EMPTY, field("value", DataType.DOUBLE), new Literal(Source.EMPTY, 1, DataType.INTEGER));
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         PlanNamedTypes.writePow(out, orig);
@@ -298,7 +297,7 @@ public void testPowSimple() throws IOException {
     }
 
     public void testAliasSimple() throws IOException {
-        var orig = new Alias(Source.EMPTY, "alias_name", field("a", DataTypes.LONG));
+        var orig = new Alias(Source.EMPTY, "alias_name", field("a", DataType.LONG));
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         PlanNamedTypes.writeAlias(out, orig);
@@ -309,7 +308,7 @@ public void testAliasSimple() throws IOException {
     }
 
     public void testLiteralSimple() throws IOException {
-        var orig = new Literal(Source.EMPTY, 1, DataTypes.INTEGER);
+        var orig = new Literal(Source.EMPTY, 1, DataType.INTEGER);
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         PlanNamedTypes.writeLiteral(out, orig);
@@ -318,7 +317,7 @@ public void testLiteralSimple() throws IOException {
     }
 
     public void testOrderSimple() throws IOException {
-        var orig = new Order(Source.EMPTY, field("val", DataTypes.INTEGER), Order.OrderDirection.ASC, Order.NullsPosition.FIRST);
+        var orig = new Order(Source.EMPTY, field("val", DataType.INTEGER), Order.OrderDirection.ASC, Order.NullsPosition.FIRST);
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         PlanNamedTypes.writeOrder(out, orig);
@@ -327,7 +326,7 @@ public void testOrderSimple() throws IOException {
     }
 
     public void testFieldSortSimple() throws IOException {
-        var orig = new EsQueryExec.FieldSort(field("val", DataTypes.LONG), Order.OrderDirection.ASC, Order.NullsPosition.FIRST);
+        var orig = new EsQueryExec.FieldSort(field("val", DataType.LONG), Order.OrderDirection.ASC, Order.NullsPosition.FIRST);
         BytesStreamOutput bso = new BytesStreamOutput();
         PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
         PlanNamedTypes.writeFieldSort(out, orig);
@@ -571,7 +570,7 @@ static Map<String, EsField> randomProperties(int depth) {
         return Map.copyOf(map);
     }
 
-    static List<DataType> DATA_TYPES = DataTypes.types().stream().toList();
+    static List<DataType> DATA_TYPES = DataType.types().stream().toList();
 
     static DataType randomDataType() {
         return DATA_TYPES.get(randomIntBetween(0, DATA_TYPES.size() - 1));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java
index a8a59f755a59f..00fb9d4943005 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutputTests.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xpack.esql.Column;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
 import org.elasticsearch.xpack.esql.session.EsqlConfigurationSerializationTests;
 
@@ -124,7 +124,7 @@ private Column randomColumn() {
             for (int i = 0; i < LEN; i++) {
                 ints.appendInt(randomInt());
             }
-            return new Column(DataTypes.INTEGER, ints.build());
+            return new Column(DataType.INTEGER, ints.build());
         }
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java
index 4674e8afc07e9..c7bec88de7082 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.core.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
@@ -151,7 +151,7 @@ public void testMissingFieldInProject() {
         var alias = as(eval.fields().get(0), Alias.class);
         var literal = as(alias.child(), Literal.class);
         assertThat(literal.fold(), is(nullValue()));
-        assertThat(literal.dataType(), is(DataTypes.KEYWORD));
+        assertThat(literal.dataType(), is(DataType.KEYWORD));
 
         var limit = as(eval.child(), Limit.class);
         var source = as(limit.child(), EsRelation.class);
@@ -206,7 +206,7 @@ public void testMissingFieldInEval() {
         var alias = as(eval.fields().get(0), Alias.class);
         var literal = as(alias.child(), Literal.class);
         assertThat(literal.fold(), is(nullValue()));
-        assertThat(literal.dataType(), is(DataTypes.INTEGER));
+        assertThat(literal.dataType(), is(DataType.INTEGER));
 
         var limit = as(eval.child(), Limit.class);
         var source = as(limit.child(), EsRelation.class);
@@ -327,7 +327,7 @@ public void testSparseDocument() throws Exception {
         Map<String, EsField> large = Maps.newLinkedHashMapWithExpectedSize(size);
         for (int i = 0; i < size; i++) {
             var name = String.format(Locale.ROOT, "field%03d", i);
-            large.put(name, new EsField(name, DataTypes.INTEGER, emptyMap(), true, false));
+            large.put(name, new EsField(name, DataType.INTEGER, emptyMap(), true, false));
         }
 
         SearchStats searchStats = statsForExistingField("field000", "field001", "field002", "field003", "field004");
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
index d25c9cdb51398..d1a352589263a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.xpack.esql.core.index.EsIndex;
 import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
@@ -88,7 +88,7 @@ public class LocalPhysicalPlanOptimizerTests extends MapperServiceTestCase {
     /**
      * Estimated size of a keyword field in bytes.
      */
-    private static final int KEYWORD_EST = EstimatesRowSize.estimateSize(DataTypes.KEYWORD);
+    private static final int KEYWORD_EST = EstimatesRowSize.estimateSize(DataType.KEYWORD);
 
     private TestPlannerOptimizer plannerOptimizer;
     private final EsqlConfiguration config;
@@ -127,8 +127,8 @@ public void init() {
                 List.of("a", "b"),
                 Map.of("", "idx"),
                 Map.ofEntries(
-                    Map.entry("a", new EsField("a", DataTypes.INTEGER, Map.of(), true)),
-                    Map.entry("b", new EsField("b", DataTypes.LONG, Map.of(), true))
+                    Map.entry("a", new EsField("a", DataType.INTEGER, Map.of(), true)),
+                    Map.entry("b", new EsField("b", DataType.LONG, Map.of(), true))
                 )
             )
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
index 4bb797faff04c..f20f6eba671d3 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
@@ -49,7 +49,6 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
@@ -150,17 +149,17 @@
 import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_SHAPE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TEXT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.VERSION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison.BinaryComparisonOperation.EQ;
 import static org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison.BinaryComparisonOperation.GT;
 import static org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison.BinaryComparisonOperation.GTE;
@@ -996,7 +995,7 @@ public void testPushDownDissectPastProject() {
 
         var keep = as(plan, Project.class);
         var dissect = as(keep.child(), Dissect.class);
-        assertThat(dissect.extractedFields(), contains(new ReferenceAttribute(Source.EMPTY, "y", DataTypes.KEYWORD)));
+        assertThat(dissect.extractedFields(), contains(new ReferenceAttribute(Source.EMPTY, "y", DataType.KEYWORD)));
     }
 
     public void testPushDownGrokPastProject() {
@@ -1009,7 +1008,7 @@ public void testPushDownGrokPastProject() {
 
         var keep = as(plan, Project.class);
         var grok = as(keep.child(), Grok.class);
-        assertThat(grok.extractedFields(), contains(new ReferenceAttribute(Source.EMPTY, "y", DataTypes.KEYWORD)));
+        assertThat(grok.extractedFields(), contains(new ReferenceAttribute(Source.EMPTY, "y", DataType.KEYWORD)));
     }
 
     public void testPushDownFilterPastProjectUsingEval() {
@@ -3769,7 +3768,7 @@ public void testCountOfLiteral() {
 
         assertThat(Expressions.names(agg.aggregates()), contains("$$COUNT$s$0", "w"));
         var countAggLiteral = as(as(Alias.unwrap(agg.aggregates().get(0)), Count.class).field(), Literal.class);
-        assertTrue(countAggLiteral.semanticEquals(new Literal(EMPTY, StringUtils.WILDCARD, DataTypes.KEYWORD)));
+        assertTrue(countAggLiteral.semanticEquals(new Literal(EMPTY, StringUtils.WILDCARD, DataType.KEYWORD)));
 
         var exprs = eval.fields();
         // s == mv_count([1,2]) * count(*)
@@ -3897,7 +3896,7 @@ private record AggOfLiteralTestCase(
             "count_distinct({}, 1234)",
             c -> new ToLong(
                 EMPTY,
-                new Coalesce(EMPTY, new MvCount(EMPTY, new MvDedupe(EMPTY, c)), List.of(new Literal(EMPTY, 0, DataTypes.INTEGER)))
+                new Coalesce(EMPTY, new MvCount(EMPTY, new MvDedupe(EMPTY, c)), List.of(new Literal(EMPTY, 0, DataType.INTEGER)))
             ),
             ints -> Arrays.stream(ints).distinct().count(),
             d -> 1L
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java
index 4b62ce1a31cb4..6ca1a638fa0ee 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
@@ -42,11 +42,11 @@
 import static org.hamcrest.Matchers.contains;
 
 public class OptimizerRulesTests extends ESTestCase {
-    private static final Literal ONE = new Literal(Source.EMPTY, 1, DataTypes.INTEGER);
-    private static final Literal TWO = new Literal(Source.EMPTY, 2, DataTypes.INTEGER);
-    private static final Literal THREE = new Literal(Source.EMPTY, 3, DataTypes.INTEGER);
-    private static final Literal FOUR = new Literal(Source.EMPTY, 4, DataTypes.INTEGER);
-    private static final Literal FIVE = new Literal(Source.EMPTY, 5, DataTypes.INTEGER);
+    private static final Literal ONE = new Literal(Source.EMPTY, 1, DataType.INTEGER);
+    private static final Literal TWO = new Literal(Source.EMPTY, 2, DataType.INTEGER);
+    private static final Literal THREE = new Literal(Source.EMPTY, 3, DataType.INTEGER);
+    private static final Literal FOUR = new Literal(Source.EMPTY, 4, DataType.INTEGER);
+    private static final Literal FIVE = new Literal(Source.EMPTY, 5, DataType.INTEGER);
 
     private static Equals equalsOf(Expression left, Expression right) {
         return new Equals(EMPTY, left, right, null);
@@ -181,7 +181,7 @@ public void testOrWithNonCombinableExpressions() {
     // Test BooleanFunctionEqualsElimination
     public void testBoolEqualsSimplificationOnExpressions() {
         OptimizerRules.BooleanFunctionEqualsElimination s = new OptimizerRules.BooleanFunctionEqualsElimination();
-        Expression exp = new GreaterThan(EMPTY, getFieldAttribute(), new Literal(EMPTY, 0, DataTypes.INTEGER), null);
+        Expression exp = new GreaterThan(EMPTY, getFieldAttribute(), new Literal(EMPTY, 0, DataType.INTEGER), null);
 
         assertEquals(exp, s.rule(new Equals(EMPTY, exp, TRUE)));
         // TODO: Replace use of QL Not with ESQL Not
@@ -225,8 +225,8 @@ public void testDualEqualsConjunction() {
     // 1 < a < 10 AND a == 10 -> FALSE
     public void testEliminateRangeByEqualsOutsideInterval() {
         FieldAttribute fa = getFieldAttribute();
-        Equals eq1 = equalsOf(fa, new Literal(EMPTY, 10, DataTypes.INTEGER));
-        Range r = rangeOf(fa, ONE, false, new Literal(EMPTY, 10, DataTypes.INTEGER), false);
+        Equals eq1 = equalsOf(fa, new Literal(EMPTY, 10, DataType.INTEGER));
+        Range r = rangeOf(fa, ONE, false, new Literal(EMPTY, 10, DataType.INTEGER), false);
 
         OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
         Expression exp = rule.rule(new And(EMPTY, eq1, r));
@@ -341,7 +341,7 @@ public void testPropagateEquals_VarEq2AndVarRangeGt1Lt3AndVarGt0AndVarNeq4() {
         FieldAttribute fa = getFieldAttribute();
         Equals eq = equalsOf(fa, TWO);
         Range range = rangeOf(fa, ONE, false, THREE, false);
-        GreaterThan gt = greaterThanOf(fa, new Literal(EMPTY, 0, DataTypes.INTEGER));
+        GreaterThan gt = greaterThanOf(fa, new Literal(EMPTY, 0, DataType.INTEGER));
         NotEquals neq = notEqualsOf(fa, FOUR);
 
         OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
@@ -480,7 +480,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() {
 
     // a == 1 AND a == 2 -> nop for date/time fields
     public void testPropagateEquals_ignoreDateTimeFields() {
-        FieldAttribute fa = TestUtils.getFieldAttribute("a", DataTypes.DATETIME);
+        FieldAttribute fa = TestUtils.getFieldAttribute("a", DataType.DATETIME);
         Equals eq1 = equalsOf(fa, ONE);
         Equals eq2 = equalsOf(fa, TWO);
         And and = new And(EMPTY, eq1, eq2);
@@ -494,7 +494,7 @@ public void testPropagateEquals_ignoreDateTimeFields() {
     public void testEliminateRangeByEqualsInInterval() {
         FieldAttribute fa = getFieldAttribute();
         Equals eq1 = equalsOf(fa, ONE);
-        Range r = rangeOf(fa, ONE, true, new Literal(EMPTY, 10, DataTypes.INTEGER), false);
+        Range r = rangeOf(fa, ONE, true, new Literal(EMPTY, 10, DataType.INTEGER), false);
 
         OptimizerRules.PropagateEquals rule = new OptimizerRules.PropagateEquals();
         Expression exp = rule.rule(new And(EMPTY, eq1, r));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
index a57754d594af9..280351b01fff7 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
@@ -48,7 +48,6 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.core.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
@@ -127,8 +126,8 @@
 import static org.elasticsearch.xpack.esql.core.expression.Expressions.names;
 import static org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.ASC;
 import static org.elasticsearch.xpack.esql.core.expression.function.scalar.FunctionTestUtils.l;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.CARTESIAN_POINT;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.GEO_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
+import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
 import static org.elasticsearch.xpack.esql.parser.ExpressionBuilder.MAX_EXPRESSION_DEPTH;
 import static org.elasticsearch.xpack.esql.parser.LogicalPlanBuilder.MAX_QUERY_DEPTH;
 import static org.elasticsearch.xpack.esql.plan.physical.AggregateExec.Mode.FINAL;
@@ -149,7 +148,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
     /**
      * Estimated size of a keyword field in bytes.
      */
-    private static final int KEYWORD_EST = EstimatesRowSize.estimateSize(DataTypes.KEYWORD);
+    private static final int KEYWORD_EST = EstimatesRowSize.estimateSize(DataType.KEYWORD);
 
     private EsqlParser parser;
     private LogicalPlanOptimizer logicalOptimizer;
@@ -240,8 +239,8 @@ private static EnrichResolution setupEnrichResolution() {
                 List.of("a", "b"),
                 Map.of("", "idx"),
                 Map.ofEntries(
-                    Map.entry("a", new EsField("a", DataTypes.INTEGER, Map.of(), true)),
-                    Map.entry("b", new EsField("b", DataTypes.LONG, Map.of(), true))
+                    Map.entry("a", new EsField("a", DataType.INTEGER, Map.of(), true)),
+                    Map.entry("b", new EsField("b", DataType.LONG, Map.of(), true))
                 )
             )
         );
@@ -254,10 +253,10 @@ private static EnrichResolution setupEnrichResolution() {
                 List.of("city", "airport", "region", "city_boundary"),
                 Map.of("", "airport_city_boundaries"),
                 Map.ofEntries(
-                    Map.entry("city", new EsField("city", DataTypes.KEYWORD, Map.of(), true)),
-                    Map.entry("airport", new EsField("airport", DataTypes.TEXT, Map.of(), false)),
-                    Map.entry("region", new EsField("region", DataTypes.TEXT, Map.of(), false)),
-                    Map.entry("city_boundary", new EsField("city_boundary", DataTypes.GEO_SHAPE, Map.of(), false))
+                    Map.entry("city", new EsField("city", DataType.KEYWORD, Map.of(), true)),
+                    Map.entry("airport", new EsField("airport", DataType.TEXT, Map.of(), false)),
+                    Map.entry("region", new EsField("region", DataType.TEXT, Map.of(), false)),
+                    Map.entry("city_boundary", new EsField("city_boundary", DataType.GEO_SHAPE, Map.of(), false))
                 )
             )
         );
@@ -269,7 +268,7 @@ private static EnrichResolution setupEnrichResolution() {
                 EnrichPolicy.MATCH_TYPE,
                 List.of("department"),
                 Map.of("", ".enrich-departments-1", "cluster_1", ".enrich-departments-2"),
-                Map.of("department", new EsField("department", DataTypes.KEYWORD, Map.of(), true))
+                Map.of("department", new EsField("department", DataType.KEYWORD, Map.of(), true))
             )
         );
         enrichResolution.addResolvedPolicy(
@@ -280,7 +279,7 @@ private static EnrichResolution setupEnrichResolution() {
                 EnrichPolicy.MATCH_TYPE,
                 List.of("department"),
                 Map.of("", ".enrich-departments-3"),
-                Map.of("department", new EsField("department", DataTypes.KEYWORD, Map.of(), true))
+                Map.of("department", new EsField("department", DataType.KEYWORD, Map.of(), true))
             )
         );
         enrichResolution.addResolvedPolicy(
@@ -291,7 +290,7 @@ private static EnrichResolution setupEnrichResolution() {
                 EnrichPolicy.MATCH_TYPE,
                 List.of("department"),
                 Map.of("cluster_1", ".enrich-departments-2"),
-                Map.of("department", new EsField("department", DataTypes.KEYWORD, Map.of(), true))
+                Map.of("department", new EsField("department", DataType.KEYWORD, Map.of(), true))
             )
         );
         enrichResolution.addResolvedPolicy(
@@ -302,7 +301,7 @@ private static EnrichResolution setupEnrichResolution() {
                 EnrichPolicy.MATCH_TYPE,
                 List.of("supervisor"),
                 Map.of("", ".enrich-supervisors-a", "cluster_1", ".enrich-supervisors-b"),
-                Map.of("supervisor", new EsField("supervisor", DataTypes.KEYWORD, Map.of(), true))
+                Map.of("supervisor", new EsField("supervisor", DataType.KEYWORD, Map.of(), true))
             )
         );
         enrichResolution.addResolvedPolicy(
@@ -313,7 +312,7 @@ private static EnrichResolution setupEnrichResolution() {
                 EnrichPolicy.MATCH_TYPE,
                 List.of("supervisor"),
                 Map.of("", ".enrich-supervisors-c"),
-                Map.of("supervisor", new EsField("supervisor", DataTypes.KEYWORD, Map.of(), true))
+                Map.of("supervisor", new EsField("supervisor", DataType.KEYWORD, Map.of(), true))
             )
         );
         enrichResolution.addResolvedPolicy(
@@ -324,7 +323,7 @@ private static EnrichResolution setupEnrichResolution() {
                 EnrichPolicy.MATCH_TYPE,
                 List.of("supervisor"),
                 Map.of("cluster_1", ".enrich-supervisors-b"),
-                Map.of("supervisor", new EsField("supervisor", DataTypes.KEYWORD, Map.of(), true))
+                Map.of("supervisor", new EsField("supervisor", DataType.KEYWORD, Map.of(), true))
             )
         );
         return enrichResolution;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
index 09dae4c739115..b24d9e6083b69 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java
@@ -44,12 +44,12 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
 import static org.elasticsearch.xpack.esql.core.expression.function.FunctionResolutionStrategy.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATE_PERIOD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.TIME_DURATION;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
index db4d54caf9943..b30b9c1598397 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
@@ -68,7 +67,7 @@
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.expression.function.FunctionResolutionStrategy.DEFAULT;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned;
 import static org.elasticsearch.xpack.esql.parser.ExpressionBuilder.breakIntoFragments;
 import static org.hamcrest.Matchers.allOf;
@@ -1129,39 +1128,39 @@ private static ReferenceAttribute referenceAttribute(String name, DataType type)
     }
 
     private static Literal integer(int i) {
-        return new Literal(EMPTY, i, DataTypes.INTEGER);
+        return new Literal(EMPTY, i, DataType.INTEGER);
     }
 
     private static Literal integers(int... ints) {
-        return new Literal(EMPTY, Arrays.stream(ints).boxed().toList(), DataTypes.INTEGER);
+        return new Literal(EMPTY, Arrays.stream(ints).boxed().toList(), DataType.INTEGER);
     }
 
     private static Literal literalLong(long i) {
-        return new Literal(EMPTY, i, DataTypes.LONG);
+        return new Literal(EMPTY, i, DataType.LONG);
     }
 
     private static Literal literalLongs(long... longs) {
-        return new Literal(EMPTY, Arrays.stream(longs).boxed().toList(), DataTypes.LONG);
+        return new Literal(EMPTY, Arrays.stream(longs).boxed().toList(), DataType.LONG);
     }
 
     private static Literal literalDouble(double d) {
-        return new Literal(EMPTY, d, DataTypes.DOUBLE);
+        return new Literal(EMPTY, d, DataType.DOUBLE);
     }
 
     private static Literal literalDoubles(double... doubles) {
-        return new Literal(EMPTY, Arrays.stream(doubles).boxed().toList(), DataTypes.DOUBLE);
+        return new Literal(EMPTY, Arrays.stream(doubles).boxed().toList(), DataType.DOUBLE);
     }
 
     private static Literal literalUnsignedLong(String ulong) {
-        return new Literal(EMPTY, asLongUnsigned(new BigInteger(ulong)), DataTypes.UNSIGNED_LONG);
+        return new Literal(EMPTY, asLongUnsigned(new BigInteger(ulong)), DataType.UNSIGNED_LONG);
     }
 
     private static Literal literalUnsignedLongs(String... ulongs) {
-        return new Literal(EMPTY, Arrays.stream(ulongs).map(s -> asLongUnsigned(new BigInteger(s))).toList(), DataTypes.UNSIGNED_LONG);
+        return new Literal(EMPTY, Arrays.stream(ulongs).map(s -> asLongUnsigned(new BigInteger(s))).toList(), DataType.UNSIGNED_LONG);
     }
 
     private static Literal literalBoolean(boolean b) {
-        return new Literal(EMPTY, b, DataTypes.BOOLEAN);
+        return new Literal(EMPTY, b, DataType.BOOLEAN);
     }
 
     private static Literal literalBooleans(boolean... booleans) {
@@ -1169,15 +1168,15 @@ private static Literal literalBooleans(boolean... booleans) {
         for (boolean b : booleans) {
             v.add(b);
         }
-        return new Literal(EMPTY, v, DataTypes.BOOLEAN);
+        return new Literal(EMPTY, v, DataType.BOOLEAN);
     }
 
     private static Literal literalString(String s) {
-        return new Literal(EMPTY, s, DataTypes.KEYWORD);
+        return new Literal(EMPTY, s, DataType.KEYWORD);
     }
 
     private static Literal literalStrings(String... strings) {
-        return new Literal(EMPTY, Arrays.asList(strings), DataTypes.KEYWORD);
+        return new Literal(EMPTY, Arrays.asList(strings), DataType.KEYWORD);
     }
 
     private void expectError(String query, String errorMessage) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java
index 11c2d9532ff16..472e1a01e0151 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java
@@ -31,7 +31,7 @@
 import static org.elasticsearch.xpack.esql.core.TestUtils.of;
 import static org.elasticsearch.xpack.esql.core.TestUtils.relation;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
-import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 import static org.hamcrest.Matchers.contains;
 
 public class QueryPlanTests extends ESTestCase {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
index d14537aede31e..fcb7d02460e94 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
@@ -61,10 +60,10 @@
 import java.util.Map;
 
 public class EvalMapperTests extends ESTestCase {
-    private static final FieldAttribute DOUBLE1 = field("foo", DataTypes.DOUBLE);
-    private static final FieldAttribute DOUBLE2 = field("bar", DataTypes.DOUBLE);
-    private static final FieldAttribute LONG = field("long", DataTypes.LONG);
-    private static final FieldAttribute DATE = field("date", DataTypes.DATETIME);
+    private static final FieldAttribute DOUBLE1 = field("foo", DataType.DOUBLE);
+    private static final FieldAttribute DOUBLE2 = field("bar", DataType.DOUBLE);
+    private static final FieldAttribute LONG = field("long", DataType.LONG);
+    private static final FieldAttribute DATE = field("date", DataType.DATETIME);
 
     private static final EsqlConfiguration TEST_CONFIG = new EsqlConfiguration(
         ZoneOffset.UTC,
@@ -81,9 +80,9 @@ public class EvalMapperTests extends ESTestCase {
 
     @ParametersFactory(argumentFormatting = "%1$s")
     public static List<Object[]> params() {
-        Literal literal = new Literal(Source.EMPTY, new BytesRef("something"), DataTypes.KEYWORD);
-        Literal datePattern = new Literal(Source.EMPTY, new BytesRef("yyyy"), DataTypes.KEYWORD);
-        Literal dateInterval = new Literal(Source.EMPTY, Duration.ofHours(1), DataTypes.TIME_DURATION);
+        Literal literal = new Literal(Source.EMPTY, new BytesRef("something"), DataType.KEYWORD);
+        Literal datePattern = new Literal(Source.EMPTY, new BytesRef("yyyy"), DataType.KEYWORD);
+        Literal dateInterval = new Literal(Source.EMPTY, Duration.ofHours(1), DataType.TIME_DURATION);
 
         List<Object[]> params = new ArrayList<>();
         for (Expression e : new Expression[] {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
index 7063853b4fce2..853096626179e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java
@@ -35,7 +35,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.index.EsIndex;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
@@ -93,9 +93,9 @@ public void testLuceneSourceOperatorHugeRowSize() throws IOException {
 
     public void testLuceneTopNSourceOperator() throws IOException {
         int estimatedRowSize = randomEstimatedRowSize(estimatedRowSizeIsHuge);
-        FieldAttribute sortField = new FieldAttribute(Source.EMPTY, "field", new EsField("field", DataTypes.INTEGER, Map.of(), true));
+        FieldAttribute sortField = new FieldAttribute(Source.EMPTY, "field", new EsField("field", DataType.INTEGER, Map.of(), true));
         EsQueryExec.FieldSort sort = new EsQueryExec.FieldSort(sortField, Order.OrderDirection.ASC, Order.NullsPosition.LAST);
-        Literal limit = new Literal(Source.EMPTY, 10, DataTypes.INTEGER);
+        Literal limit = new Literal(Source.EMPTY, 10, DataType.INTEGER);
         LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan(
             new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, limit, List.of(sort), estimatedRowSize)
         );
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
index 8134dc16aedf6..a8b371bcdd608 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeSubclassTests;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow;
@@ -158,7 +157,7 @@ static String randomGrokPattern() {
         );
     }
 
-    static List<DataType> DATA_TYPES = DataTypes.types().stream().toList();
+    static List<DataType> DATA_TYPES = DataType.types().stream().toList();
 
     static EsQueryExec.FieldSort randomFieldSort() {
         return new EsQueryExec.FieldSort(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java
index 9eaef21421f55..7dca73219d6a1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.session.EsqlIndexResolver;
 
@@ -25,18 +24,18 @@
 public class EsqlDataTypeRegistryTests extends ESTestCase {
 
     public void testCounter() {
-        resolve("long", TimeSeriesParams.MetricType.COUNTER, DataTypes.COUNTER_LONG);
-        resolve("integer", TimeSeriesParams.MetricType.COUNTER, DataTypes.COUNTER_INTEGER);
-        resolve("double", TimeSeriesParams.MetricType.COUNTER, DataTypes.COUNTER_DOUBLE);
+        resolve("long", TimeSeriesParams.MetricType.COUNTER, DataType.COUNTER_LONG);
+        resolve("integer", TimeSeriesParams.MetricType.COUNTER, DataType.COUNTER_INTEGER);
+        resolve("double", TimeSeriesParams.MetricType.COUNTER, DataType.COUNTER_DOUBLE);
 
     }
 
     public void testGauge() {
-        resolve("long", TimeSeriesParams.MetricType.GAUGE, DataTypes.LONG);
+        resolve("long", TimeSeriesParams.MetricType.GAUGE, DataType.LONG);
     }
 
     public void testLong() {
-        resolve("long", null, DataTypes.LONG);
+        resolve("long", null, DataType.LONG);
     }
 
     private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, DataType expected) {

From fa930cb7b3754bcba36aec3592edb632e2cd540e Mon Sep 17 00:00:00 2001
From: Fang Xing <155562079+fang-xing-esql@users.noreply.github.com>
Date: Tue, 4 Jun 2024 15:38:45 -0400
Subject: [PATCH 142/208] [ES|QL] Convert string to datetime when the other
 size of an arithmetic operator is date_period or time_duration (#108455)

* convert string to datetime when the other side of binary operator is temporal amount
---
 docs/changelog/108455.yaml                    |  6 +++
 .../src/main/resources/date.csv-spec          | 49 ++++++++++++++++++
 .../xpack/esql/action/EsqlCapabilities.java   |  6 +++
 .../xpack/esql/analysis/Analyzer.java         | 34 ++++++++-----
 .../DateTimeArithmeticOperation.java          |  2 +-
 .../xpack/esql/analysis/AnalyzerTests.java    | 50 +++++++++++++++++++
 6 files changed, 134 insertions(+), 13 deletions(-)
 create mode 100644 docs/changelog/108455.yaml

diff --git a/docs/changelog/108455.yaml b/docs/changelog/108455.yaml
new file mode 100644
index 0000000000000..8397af7b07cf1
--- /dev/null
+++ b/docs/changelog/108455.yaml
@@ -0,0 +1,6 @@
+pr: 108455
+summary: "[ES|QL] Convert string to datetime when the other size of an arithmetic\
+  \ operator is `date_period` or `time_duration`"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
index 63421aec35665..776cc2f95f465 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
@@ -1122,3 +1122,52 @@ emp_no:integer | birth_date:datetime
 10003          | 1959-12-03T00:00:00Z
 10008          | 1958-02-19T00:00:00Z
 ;
+
+implicitCastingArithmeticOperationAdd
+required_capability: string_literal_auto_casting_to_datetime_add_sub
+from employees
+| eval a =  1 day + "2024-01-01", b = 1 year + "2024-04-01" + 1 month, c = "2024-01-01" + 3600 seconds,
+       d = "2024-04-01" + (1 year + 1 day)
+| keep a, b, c, d
+| limit 1
+;
+
+a:datetime | b:datetime | c:datetime               | d:datetime
+2024-01-02 | 2025-05-01 | 2024-01-01T01:00:00.000Z | 2025-04-02
+;
+
+implicitCastingArithmeticOperationSub
+required_capability: string_literal_auto_casting_to_datetime_add_sub
+from employees
+| eval a =  "2024-01-01" - 1 day, b = "2024-04-01" - 1 month, c = "2024-01-01" - 3600 seconds,
+       d = "2024-04-01" - (1 year + 1 day)
+| keep a, b, c, d
+| limit 1
+;
+
+a:datetime | b:datetime | c:datetime               | d:datetime
+2023-12-31 | 2024-03-01 | 2023-12-31T23:00:00.000Z | 2023-03-31
+;
+
+implicitCastingArithmeticOperationAddSub
+required_capability: string_literal_auto_casting_to_datetime_add_sub
+from employees
+| eval a = 1 month + "2024-01-01" - 1 day, b = - 1 year + "2024-04-01" + 1 month,
+       c = 1 hour + "2024-01-01" - 3600 seconds, d = "2024-04-01" - (1 year + 1 day)
+| keep a, b, c, d
+| limit 1
+;
+
+a:datetime | b:datetime | c:datetime | d:datetime
+2024-01-31 | 2023-05-01 | 2024-01-01 | 2023-03-31
+;
+
+temporalAmountWithNulls
+from employees
+| eval a = to_dt(null) - 1 day
+| keep a
+| limit 1;
+
+a:datetime
+null
+;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index 61134da3ecb85..907422b179657 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -59,6 +59,11 @@ public class EsqlCapabilities {
 
     public static final Set<String> CAPABILITIES = capabilities();
 
+    /**
+     * Cast string literals to datetime in addition and subtraction when the other side is a date or time interval.
+     */
+    public static final String STRING_LITERAL_AUTO_CASTING_TO_DATETIME_ADD_SUB = "string_literal_auto_casting_to_datetime_add_sub";
+
     private static Set<String> capabilities() {
         List<String> caps = new ArrayList<>();
         caps.add(FN_CBRT);
@@ -81,6 +86,7 @@ private static Set<String> capabilities() {
         for (NodeFeature feature : new EsqlFeatures().getHistoricalFeatures().keySet()) {
             caps.add(cap(feature));
         }
+        caps.add(STRING_LITERAL_AUTO_CASTING_TO_DATETIME_ADD_SUB);
         return Set.copyOf(caps);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index 90b2197f11dae..c8d3207d016e4 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -57,6 +57,7 @@
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
 import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.DateTimeArithmeticOperation;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
@@ -107,6 +108,7 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
 import static org.elasticsearch.xpack.esql.stats.FeatureMetric.LIMIT;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
 
 public class Analyzer extends ParameterizedRuleExecutor<LogicalPlan, AnalyzerContext> {
     // marker list of attributes for plans that do not have any concrete fields to return, but have other computed columns to return
@@ -850,19 +852,23 @@ private static Expression processBinaryOperator(BinaryOperator<?, ?, ?, ?> o) {
             DataType targetDataType = DataType.NULL;
             Expression from = Literal.NULL;
 
-            if (left.dataType() == KEYWORD
-                && left.foldable()
-                && (supportsImplicitCasting(right.dataType()))
-                && ((left instanceof EsqlScalarFunction) == false)) {
-                targetDataType = right.dataType();
-                from = left;
+            if (left.dataType() == KEYWORD && left.foldable() && (left instanceof EsqlScalarFunction == false)) {
+                if (supportsImplicitCasting(right.dataType())) {
+                    targetDataType = right.dataType();
+                    from = left;
+                } else if (supportsImplicitTemporalCasting(right, o)) {
+                    targetDataType = DATETIME;
+                    from = left;
+                }
             }
-            if (right.dataType() == KEYWORD
-                && right.foldable()
-                && (supportsImplicitCasting(left.dataType()))
-                && ((right instanceof EsqlScalarFunction) == false)) {
-                targetDataType = left.dataType();
-                from = right;
+            if (right.dataType() == KEYWORD && right.foldable() && (right instanceof EsqlScalarFunction == false)) {
+                if (supportsImplicitCasting(left.dataType())) {
+                    targetDataType = left.dataType();
+                    from = right;
+                } else if (supportsImplicitTemporalCasting(left, o)) {
+                    targetDataType = DATETIME;
+                    from = right;
+                }
             }
             if (from != Literal.NULL) {
                 Expression e = castStringLiteral(from, targetDataType);
@@ -896,6 +902,10 @@ private static Expression processIn(In in) {
             return childrenChanged ? in.replaceChildren(newChildren) : in;
         }
 
+        private static boolean supportsImplicitTemporalCasting(Expression e, BinaryOperator<?, ?, ?, ?> o) {
+            return isTemporalAmount(e.dataType()) && (o instanceof DateTimeArithmeticOperation);
+        }
+
         private static boolean supportsImplicitCasting(DataType type) {
             return type == DATETIME || type == IP || type == VERSION || type == BOOLEAN;
         }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
index 2f2af5c0ff8d7..04a7b8a6067bd 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
@@ -29,7 +29,7 @@
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isDateTimeOrTemporal;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isTemporalAmount;
 
-abstract class DateTimeArithmeticOperation extends EsqlArithmeticOperation {
+public abstract class DateTimeArithmeticOperation extends EsqlArithmeticOperation {
     /** Arithmetic (quad) function. */
     interface DatetimeArithmeticEvaluator {
         ExpressionEvaluator.Factory apply(Source source, ExpressionEvaluator.Factory expressionEvaluator, TemporalAmount temporalAmount);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index fb3c5d737944e..8adfc5813b855 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -1931,6 +1931,56 @@ public void testLookup() {
         assertThat(e.getMessage(), containsString("LOOKUP not yet supported"));
     }
 
+    public void testImplicitCasting() {
+        var e = expectThrows(VerificationException.class, () -> analyze("""
+             from test | eval x = concat("2024", "-04", "-01") + 1 day
+            """));
+
+        assertThat(
+            e.getMessage(),
+            containsString("first argument of [concat(\"2024\", \"-04\", \"-01\") + 1 day] must be [datetime or numeric]")
+        );
+
+        e = expectThrows(VerificationException.class, () -> analyze("""
+             from test | eval x = to_string(null) - 1 day
+            """));
+
+        assertThat(e.getMessage(), containsString("first argument of [to_string(null) - 1 day] must be [datetime or numeric]"));
+
+        e = expectThrows(VerificationException.class, () -> analyze("""
+             from test | eval x = concat("2024", "-04", "-01") + "1 day"
+            """));
+
+        assertThat(
+            e.getMessage(),
+            containsString("first argument of [concat(\"2024\", \"-04\", \"-01\") + \"1 day\"] must be [datetime or numeric]")
+        );
+
+        e = expectThrows(VerificationException.class, () -> analyze("""
+             from test | eval x = 1 year - "2024-01-01" + 1 day
+            """));
+
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "arguments are in unsupported order: cannot subtract a [DATETIME] value [\"2024-01-01\"] "
+                    + "from a [DATE_PERIOD] amount [1 year]"
+            )
+        );
+
+        e = expectThrows(VerificationException.class, () -> analyze("""
+             from test | eval x = "2024-01-01" - 1 day - "2023-12-31"
+            """));
+
+        assertThat(e.getMessage(), containsString("[-] has arguments with incompatible types [datetime] and [datetime]"));
+
+        e = expectThrows(VerificationException.class, () -> analyze("""
+             from test | eval x = "2024-01-01" - 1 day + "2023-12-31"
+            """));
+
+        assertThat(e.getMessage(), containsString("[+] has arguments with incompatible types [datetime] and [datetime]"));
+    }
+
     private void verifyUnsupported(String query, String errorMessage) {
         verifyUnsupported(query, errorMessage, "mapping-multi-field-variation.json");
     }

From f1153b1f8d742a5d6bb0741ad4b4a3aaa18b6468 Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Tue, 4 Jun 2024 12:45:08 -0700
Subject: [PATCH 143/208] Revert "Ensure synthetic source and dv codec are
 enabled with logs index mode. (#109269)" (#109365)

This reverts commit 4161e4d2e247c5a65cfbac84aab1813e03fa5f51.
---
 .../test/aggregations/time_series.yml         |  6 +--
 .../rest-api-spec/test/logsdb/10_settings.yml |  4 --
 .../cluster/metadata/IndexMetadata.java       |  7 ++-
 .../index/mapper/SourceFieldMapper.java       | 41 ++++------------
 .../index/LogsIndexModeTests.java             | 31 +++++-------
 .../index/codec/PerFieldMapperCodecTests.java | 47 ++++---------------
 .../index/mapper/SourceFieldMapperTests.java  |  7 ---
 .../index/mapper/MapperServiceTestCase.java   |  5 --
 8 files changed, 34 insertions(+), 114 deletions(-)

diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
index 1703d4908a753..7800923ab1580 100644
--- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
+++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
@@ -294,11 +294,11 @@ setup:
 ---
 "Configure with no synthetic source":
   - requires:
-      cluster_features: ["gte_v8.15.0"]
-      reason: "Error message changed in 8.15.0"
+      cluster_features: ["gte_v8.9.0"]
+      reason: "Error message fix in 8.9"
 
   - do:
-      catch: '/Indices with with index mode \[time_series\] only support synthetic source/'
+      catch: '/Time series indices only support synthetic source./'
       indices.create:
         index: tsdb_error
         body:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
index 128903f4faac8..95075da20fe5e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
@@ -77,10 +77,6 @@ create logs index:
   - is_true: test
   - match: { test.settings.index.mode: "logs" }
 
-  - do:
-      indices.get_mapping:
-        index: test
-  - match: { test.mappings._source.mode: synthetic }
 
 ---
 using default timestamp field mapping:
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
index 64809c963cb6d..681dcb3e314e3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -2267,9 +2267,8 @@ IndexMetadata build(boolean repair) {
             }
 
             final boolean isSearchableSnapshot = SearchableSnapshotsSettings.isSearchableSnapshotStore(settings);
-            String indexModeString = settings.get(IndexSettings.MODE.getKey());
-            final IndexMode indexMode = indexModeString != null ? IndexMode.fromString(indexModeString.toLowerCase(Locale.ROOT)) : null;
-            final boolean isTsdb = indexMode == IndexMode.TIME_SERIES;
+            final String indexMode = settings.get(IndexSettings.MODE.getKey());
+            final boolean isTsdb = indexMode != null && IndexMode.TIME_SERIES.getName().equals(indexMode.toLowerCase(Locale.ROOT));
             return new IndexMetadata(
                 new Index(index, uuid),
                 version,
@@ -2309,7 +2308,7 @@ IndexMetadata build(boolean repair) {
                 AutoExpandReplicas.SETTING.get(settings),
                 isSearchableSnapshot,
                 isSearchableSnapshot && settings.getAsBoolean(SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY, false),
-                indexMode,
+                isTsdb ? IndexMode.TIME_SERIES : null,
                 isTsdb ? IndexSettings.TIME_SERIES_START_TIME.get(settings) : null,
                 isTsdb ? IndexSettings.TIME_SERIES_END_TIME.get(settings) : null,
                 SETTING_INDEX_VERSION_COMPATIBILITY.get(settings),
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index 045a17409877b..b819ffb0ef6ad 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -69,14 +69,6 @@ private enum Mode {
         IndexMode.TIME_SERIES
     );
 
-    private static final SourceFieldMapper LOGS_DEFAULT = new SourceFieldMapper(
-        Mode.SYNTHETIC,
-        Explicit.IMPLICIT_TRUE,
-        Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY,
-        IndexMode.LOGS
-    );
-
     /*
      * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below
      * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version.
@@ -164,8 +156,7 @@ protected Parameter<?>[] getParameters() {
 
         private boolean isDefault() {
             Mode m = mode.get();
-            if (m != null
-                && (((indexMode != null && indexMode.isSyntheticSourceEnabled() && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
+            if (m != null && (((indexMode == IndexMode.TIME_SERIES && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
                 return false;
             }
             return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty();
@@ -174,19 +165,15 @@ private boolean isDefault() {
         @Override
         public SourceFieldMapper build() {
             if (enabled.getValue().explicit()) {
-                if (indexMode != null && indexMode.isSyntheticSourceEnabled()) {
-                    throw new MapperParsingException("Indices with with index mode [" + indexMode + "] only support synthetic source");
+                if (indexMode == IndexMode.TIME_SERIES) {
+                    throw new MapperParsingException("Time series indices only support synthetic source");
                 }
                 if (mode.get() != null) {
                     throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
                 }
             }
             if (isDefault()) {
-                return switch (indexMode) {
-                    case TIME_SERIES -> TSDB_DEFAULT;
-                    case LOGS -> LOGS_DEFAULT;
-                    default -> DEFAULT;
-                };
+                return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT;
             }
             if (supportsNonDefaultParameterValues == false) {
                 List<String> disallowed = new ArrayList<>();
@@ -225,21 +212,10 @@ public SourceFieldMapper build() {
 
     }
 
-    public static final TypeParser PARSER = new ConfigurableTypeParser(c -> {
-        var indexMode = c.getIndexSettings().getMode();
-        if (indexMode.isSyntheticSourceEnabled()) {
-            if (indexMode == IndexMode.TIME_SERIES) {
-                if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) {
-                    return TSDB_DEFAULT;
-                } else {
-                    return TSDB_LEGACY_DEFAULT;
-                }
-            } else if (indexMode == IndexMode.LOGS) {
-                return LOGS_DEFAULT;
-            }
-        }
-        return DEFAULT;
-    },
+    public static final TypeParser PARSER = new ConfigurableTypeParser(
+        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES
+            ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT
+            : DEFAULT,
         c -> new Builder(
             c.getIndexSettings().getMode(),
             c.getSettings(),
@@ -347,7 +323,6 @@ public void preParse(DocumentParserContext context) throws IOException {
         final BytesReference adaptedSource = applyFilters(originalSource, contentType);
 
         if (adaptedSource != null) {
-            assert indexMode == null || indexMode.isSyntheticSourceEnabled() == false;
             final BytesRef ref = adaptedSource.toBytesRef();
             context.doc().add(new StoredField(fieldType().name(), ref.bytes, ref.offset, ref.length));
         }
diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
index caddc7d5ea5af..fd73a8c9f8f52 100644
--- a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
@@ -10,13 +10,12 @@
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.index.mapper.MapperServiceTestCase;
+import org.hamcrest.Matchers;
 
-import static org.hamcrest.Matchers.equalTo;
-
-public class LogsIndexModeTests extends ESTestCase {
+public class LogsIndexModeTests extends MapperServiceTestCase {
     public void testLogsIndexModeSetting() {
-        assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGS));
+        assertThat(IndexSettings.MODE.get(buildSettings()), Matchers.equalTo(IndexMode.LOGS));
     }
 
     public void testSortField() {
@@ -25,10 +24,8 @@ public void testSortField() {
             .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
-        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
-        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
     }
 
     public void testSortMode() {
@@ -38,11 +35,9 @@ public void testSortMode() {
             .put(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "max")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
-        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
-        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("max", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey())));
+        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("max", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey())));
     }
 
     public void testSortOrder() {
@@ -52,11 +47,9 @@ public void testSortOrder() {
             .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
-        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
-        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("desc", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey())));
+        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("desc", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey())));
     }
 
     public void testSortMissing() {
@@ -66,11 +59,9 @@ public void testSortMissing() {
             .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
-        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
-        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("_last", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey())));
+        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("_last", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey())));
     }
 
     private Settings buildSettings() {
diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
index 525fa31673494..74657842488b5 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat;
@@ -62,28 +61,6 @@ public class PerFieldMapperCodecTests extends ESTestCase {
         }
         """;
 
-    private static final String MAPPING_3 = """
-        {
-            "_data_stream_timestamp": {
-                "enabled": true
-            },
-            "properties": {
-                "@timestamp": {
-                    "type": "date"
-                },
-                "hostname": {
-                    "type": "keyword"
-                },
-                "response_size": {
-                    "type": "long"
-                },
-                "message": {
-                    "type": "text"
-                }
-            }
-        }
-        """;
-
     public void testUseBloomFilter() throws IOException {
         PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, randomBoolean(), false);
         assertThat(perFieldMapperCodec.useBloomFilter("_id"), is(true));
@@ -126,13 +103,13 @@ public void testDoNotUseES87TSDBEncodingForTimestampFieldNonTimeSeriesIndex() th
     }
 
     public void testEnableES87TSDBCodec() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.TIME_SERIES, MAPPING_1);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, true, MAPPING_1);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
     }
 
     public void testDisableES87TSDBCodec() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, IndexMode.TIME_SERIES, MAPPING_1);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, true, MAPPING_1);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
     }
@@ -167,37 +144,31 @@ private PerFieldFormatSupplier createFormatSupplier(boolean timestampField, bool
     }
 
     public void testUseES87TSDBEncodingSettingDisabled() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, IndexMode.TIME_SERIES, MAPPING_2);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, true, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
     }
 
     public void testUseTimeSeriesModeDisabledCodecDisabled() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.STANDARD, MAPPING_2);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, false, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
     }
 
     public void testUseTimeSeriesModeAndCodecEnabled() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.TIME_SERIES, MAPPING_2);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, true, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
     }
 
-    public void testLogsIndexMode() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGS, MAPPING_3);
-        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
-        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("hostname")), is(true));
-        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("response_size")), is(true));
-    }
-
-    private PerFieldFormatSupplier createFormatSupplier(boolean enableES87TSDBCodec, IndexMode mode, String mapping) throws IOException {
+    private PerFieldFormatSupplier createFormatSupplier(boolean enableES87TSDBCodec, boolean timeSeries, String mapping)
+        throws IOException {
         Settings.Builder settings = Settings.builder();
-        settings.put(IndexSettings.MODE.getKey(), mode);
-        if (mode == IndexMode.TIME_SERIES) {
+        if (timeSeries) {
+            settings.put(IndexSettings.MODE.getKey(), "time_series");
             settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field");
         }
         settings.put(IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING.getKey(), enableES87TSDBCodec);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
index d0350c1d92a83..802a18645eab6 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
@@ -243,13 +243,6 @@ public void testSyntheticSourceInTimeSeries() throws IOException {
         assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString());
     }
 
-    public void testSyntheticSourceWithLogsIndexMode() throws IOException {
-        XContentBuilder mapping = fieldMapping(b -> { b.field("type", "keyword"); });
-        DocumentMapper mapper = createLogsModeDocumentMapper(mapping);
-        assertTrue(mapper.sourceMapper().isSynthetic());
-        assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString());
-    }
-
     public void testSupportsNonDefaultParameterValues() throws IOException {
         Settings settings = Settings.builder().put(SourceFieldMapper.LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, false).build();
         {
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
index ca49c1b8a6f7d..7d13e33be0db7 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
@@ -148,11 +148,6 @@ protected final DocumentMapper createTimeSeriesModeDocumentMapper(XContentBuilde
         return createMapperService(settings, mappings).documentMapper();
     }
 
-    protected final DocumentMapper createLogsModeDocumentMapper(XContentBuilder mappings) throws IOException {
-        Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), "logs").build();
-        return createMapperService(settings, mappings).documentMapper();
-    }
-
     protected final DocumentMapper createDocumentMapper(IndexVersion version, XContentBuilder mappings) throws IOException {
         return createMapperService(version, mappings).documentMapper();
     }

From e1eedf5dfaf91d1bdb6687667e3655d5a4a849bb Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Tue, 4 Jun 2024 21:56:18 +0200
Subject: [PATCH 144/208] Remove redundant Map.Entry implementation in
 o.e.c.util.Maps (#109336)

This implementation of an entry that allows nulls is redundant, turns
out the JDK had one all along.
---
 .../elasticsearch/cluster/DiffableUtils.java  | 10 +++---
 .../common/collect/ImmutableOpenMap.java      |  8 ++---
 .../org/elasticsearch/common/util/Maps.java   | 35 -------------------
 3 files changed, 10 insertions(+), 43 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java b/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
index d2ebab48142d6..061da87b00f69 100644
--- a/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
+++ b/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
@@ -13,9 +13,9 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
-import org.elasticsearch.common.util.Maps;
 
 import java.io.IOException;
+import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -154,7 +154,9 @@ private static <K, T, M extends Map<K, T>> MapDiff<K, T, M> createDiff(
                 inserts++;
             } else if (entry.getValue().equals(previousValue) == false) {
                 if (valueSerializer.supportsDiffableValues()) {
-                    diffs.add(new Maps.ImmutableEntry<>(entry.getKey(), valueSerializer.diff(entry.getValue(), previousValue)));
+                    diffs.add(
+                        new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), valueSerializer.diff(entry.getValue(), previousValue))
+                    );
                 } else {
                     upserts.add(entry);
                 }
@@ -308,14 +310,14 @@ private MapDiff(
             for (int i = 0; i < diffsCount; i++) {
                 K key = keySerializer.readKey(in);
                 Diff<T> diff = valueSerializer.readDiff(in, key);
-                diffs.add(new Maps.ImmutableEntry<>(key, diff));
+                diffs.add(new AbstractMap.SimpleImmutableEntry<>(key, diff));
             }
             int upsertsCount = in.readVInt();
             upserts = upsertsCount == 0 ? List.of() : new ArrayList<>(upsertsCount);
             for (int i = 0; i < upsertsCount; i++) {
                 K key = keySerializer.readKey(in);
                 T newValue = valueSerializer.read(in, key);
-                upserts.add(new Maps.ImmutableEntry<>(key, newValue));
+                upserts.add(new AbstractMap.SimpleImmutableEntry<>(key, newValue));
             }
             this.builderCtor = builderCtor;
         }
diff --git a/server/src/main/java/org/elasticsearch/common/collect/ImmutableOpenMap.java b/server/src/main/java/org/elasticsearch/common/collect/ImmutableOpenMap.java
index 33d8fbf99f31f..cce61f5ff55e3 100644
--- a/server/src/main/java/org/elasticsearch/common/collect/ImmutableOpenMap.java
+++ b/server/src/main/java/org/elasticsearch/common/collect/ImmutableOpenMap.java
@@ -14,8 +14,6 @@
 import com.carrotsearch.hppc.procedures.ObjectObjectProcedure;
 import com.carrotsearch.hppc.procedures.ObjectProcedure;
 
-import org.elasticsearch.common.util.Maps;
-
 import java.util.AbstractCollection;
 import java.util.AbstractMap;
 import java.util.AbstractSet;
@@ -146,7 +144,7 @@ public boolean isEmpty() {
 
         @Override
         public Iterator<Map.Entry<KType, VType>> iterator() {
-            return Iterators.map(map.iterator(), c -> new Maps.ImmutableEntry<>(c.key, c.value));
+            return Iterators.map(map.iterator(), c -> new AbstractMap.SimpleImmutableEntry<>(c.key, c.value));
         }
 
         @Override
@@ -156,7 +154,9 @@ public Spliterator<Map.Entry<KType, VType>> spliterator() {
 
         @Override
         public void forEach(Consumer<? super Map.Entry<KType, VType>> action) {
-            map.forEach((Consumer<ObjectObjectCursor<KType, VType>>) c -> action.accept(new Maps.ImmutableEntry<>(c.key, c.value)));
+            map.forEach(
+                (Consumer<ObjectObjectCursor<KType, VType>>) c -> action.accept(new AbstractMap.SimpleImmutableEntry<>(c.key, c.value))
+            );
         }
 
         @SuppressWarnings("unchecked")
diff --git a/server/src/main/java/org/elasticsearch/common/util/Maps.java b/server/src/main/java/org/elasticsearch/common/util/Maps.java
index fc911793711b7..8e36cd250f7b7 100644
--- a/server/src/main/java/org/elasticsearch/common/util/Maps.java
+++ b/server/src/main/java/org/elasticsearch/common/util/Maps.java
@@ -329,39 +329,4 @@ public static <K, V1, V2> Map<K, V2> transformValues(Map<K, V1> source, Function
         return copy;
     }
 
-    /**
-     * An immutable implementation of {@link Map.Entry}.
-     * Unlike {@code Map.entry(...)} this implementation permits null key and value.
-     */
-    public record ImmutableEntry<KType, VType>(KType key, VType value) implements Map.Entry<KType, VType> {
-
-        @Override
-        public KType getKey() {
-            return key;
-        }
-
-        @Override
-        public VType getValue() {
-            return value;
-        }
-
-        @Override
-        public VType setValue(VType value) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        @SuppressWarnings("rawtypes")
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if ((o instanceof Map.Entry) == false) return false;
-            Map.Entry that = (Map.Entry) o;
-            return Objects.equals(key, that.getKey()) && Objects.equals(value, that.getValue());
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hashCode(key) ^ Objects.hashCode(value);
-        }
-    }
 }

From 1b74fb7e60ee1ae7bf362b9a35a7afb59aaa8ded Mon Sep 17 00:00:00 2001
From: Luigi Dell'Aquila <luigi.dellaquila@gmail.com>
Date: Tue, 4 Jun 2024 21:57:57 +0200
Subject: [PATCH 145/208] ES|QL: Fix flaky MV_APPEND test (#109368)

Fixing a test that failed randomly due to non-deterministic result
order.
---
 .../esql/qa/testFixtures/src/main/resources/floats.csv-spec    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec
index e88daa63ec557..66f4e9a33ceff 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec
@@ -534,7 +534,8 @@ FROM employees
        i2 = mv_append(emp_no, salary_change.int),
        i3 = mv_append(emp_no, emp_no),  
        s = mv_append(salary_change.keyword, salary_change.keyword)
-| KEEP emp_no, salary_change, d, i, i2, i3, s;
+| KEEP emp_no, salary_change, d, i, i2, i3, s
+| SORT emp_no;
 
 emp_no:integer | salary_change:double    | d:double                                       | i:integer              | i2:integer         | i3:integer     | s:keyword
 10008          | [-2.92,0.75,3.54,12.68] | [-2.92,0.75,3.54,12.68,-2.92,0.75,3.54,12.68]  | [-2,0,3,12,-2,0,3,12]  | [10008,-2,0,3,12]  | [10008, 10008] | [-2.92,0.75,12.68,3.54,-2.92,0.75,12.68,3.54]

From a6e9e906535db2b8958462d92391df7346eee977 Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Tue, 4 Jun 2024 16:06:56 -0400
Subject: [PATCH 146/208] Renable test suite, muting bad test (#109366)

---
 muted-tests.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 373c3e8cc65c4..147267cb1d197 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -41,14 +41,12 @@ tests:
 - class: org.elasticsearch.xpack.ccr.LocalIndexFollowingIT
   method: testRemoveRemoteConnection
   issue: https://github.com/elastic/elasticsearch/issues/109163
-- class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
-  method: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/109266
 - class: "org.elasticsearch.index.engine.frozen.FrozenIndexIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109315"
   method: "testTimestampFieldTypeExposedByAllIndicesServices"
 - class: "org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109318"
+  method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)}"
 - class: "org.elasticsearch.upgrades.AggregationsIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109322"
   method: "testTerms"

From 96e929512b18c310ec22dee4b3d1ed1e696b57e6 Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Tue, 4 Jun 2024 16:18:47 -0400
Subject: [PATCH 147/208] Fix mv_append (#109369)

We had a race condition when merging
https://github.com/elastic/elasticsearch/pull/109227. This fixes
`mv_append` to point to the correct classes.
---
 .../multivalue/MvAppendBooleanEvaluator.java  |   2 +-
 .../multivalue/MvAppendBytesRefEvaluator.java |   2 +-
 .../multivalue/MvAppendDoubleEvaluator.java   |   2 +-
 .../multivalue/MvAppendIntEvaluator.java      |   2 +-
 .../multivalue/MvAppendLongEvaluator.java     |   2 +-
 .../function/scalar/multivalue/MvAppend.java  |   3 +-
 .../scalar/multivalue/MvAppendTests.java      | 138 +++++++++---------
 7 files changed, 75 insertions(+), 76 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java
index c59d915fa2fe4..83e49464fc43a 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java
@@ -30,10 +30,10 @@ public final class MvAppendBooleanEvaluator implements EvalOperator.ExpressionEv
 
   public MvAppendBooleanEvaluator(Source source, EvalOperator.ExpressionEvaluator field1,
       EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field1 = field1;
     this.field2 = field2;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java
index c650a803a6dd4..6baea4de982ff 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java
@@ -30,10 +30,10 @@ public final class MvAppendBytesRefEvaluator implements EvalOperator.ExpressionE
 
   public MvAppendBytesRefEvaluator(Source source, EvalOperator.ExpressionEvaluator field1,
       EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field1 = field1;
     this.field2 = field2;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java
index 07108c0522b8d..f714fcefac8c8 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java
@@ -30,10 +30,10 @@ public final class MvAppendDoubleEvaluator implements EvalOperator.ExpressionEva
 
   public MvAppendDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator field1,
       EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field1 = field1;
     this.field2 = field2;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java
index 99c5c9798a0aa..1fbbdf81323bb 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java
@@ -30,10 +30,10 @@ public final class MvAppendIntEvaluator implements EvalOperator.ExpressionEvalua
 
   public MvAppendIntEvaluator(Source source, EvalOperator.ExpressionEvaluator field1,
       EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field1 = field1;
     this.field2 = field2;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java
index 340a9747b14aa..14f27ae88964b 100644
--- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java
@@ -30,10 +30,10 @@ public final class MvAppendLongEvaluator implements EvalOperator.ExpressionEvalu
 
   public MvAppendLongEvaluator(Source source, EvalOperator.ExpressionEvaluator field1,
       EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) {
-    this.warnings = new Warnings(source);
     this.field1 = field1;
     this.field2 = field2;
     this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
   }
 
   @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java
index 21da054122aeb..1f37c15ecfc43 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
 import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
@@ -115,7 +114,7 @@ protected TypeResolution resolveType() {
             return resolution;
         }
         dataType = field1.dataType();
-        if (dataType == DataTypes.NULL) {
+        if (dataType == DataType.NULL) {
             dataType = field2.dataType();
             return isType(field2, EsqlDataTypes::isRepresentable, sourceText(), SECOND, "representable");
         }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
index 6d43a1087ad88..07dab82c50607 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.geo.ShapeTestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
@@ -50,245 +50,245 @@ protected Expression build(Source source, List<Expression> args) {
     }
 
     private static void booleans(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.BOOLEAN, DataTypes.BOOLEAN), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.BOOLEAN, DataType.BOOLEAN), () -> {
             List<Boolean> field1 = randomList(1, 10, () -> randomBoolean());
             List<Boolean> field2 = randomList(1, 10, () -> randomBoolean());
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.BOOLEAN, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.BOOLEAN, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.BOOLEAN, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.BOOLEAN, "field2")
                 ),
                 "MvAppendBooleanEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.BOOLEAN,
+                DataType.BOOLEAN,
                 equalTo(result)
             );
         }));
     }
 
     private static void ints(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.INTEGER, DataType.INTEGER), () -> {
             List<Integer> field1 = randomList(1, 10, () -> randomInt());
             List<Integer> field2 = randomList(1, 10, () -> randomInt());
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.INTEGER, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.INTEGER, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.INTEGER, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.INTEGER, "field2")
                 ),
                 "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 equalTo(result)
             );
         }));
     }
 
     private static void longs(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.LONG, DataTypes.LONG), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.LONG, DataType.LONG), () -> {
             List<Long> field1 = randomList(1, 10, () -> randomLong());
             List<Long> field2 = randomList(1, 10, () -> randomLong());
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.LONG, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.LONG, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.LONG, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.LONG, "field2")
                 ),
                 "MvAppendLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.LONG,
+                DataType.LONG,
                 equalTo(result)
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.DATETIME, DataTypes.DATETIME), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.DATETIME, DataType.DATETIME), () -> {
             List<Long> field1 = randomList(1, 10, () -> randomLong());
             List<Long> field2 = randomList(1, 10, () -> randomLong());
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.DATETIME, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.DATETIME, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.DATETIME, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.DATETIME, "field2")
                 ),
                 "MvAppendLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.DATETIME,
+                DataType.DATETIME,
                 equalTo(result)
             );
         }));
     }
 
     private static void doubles(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.DOUBLE, DataTypes.DOUBLE), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.DOUBLE, DataType.DOUBLE), () -> {
             List<Double> field1 = randomList(1, 10, () -> randomDouble());
             List<Double> field2 = randomList(1, 10, () -> randomDouble());
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.DOUBLE, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.DOUBLE, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.DOUBLE, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.DOUBLE, "field2")
                 ),
                 "MvAppendDoubleEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.DOUBLE,
+                DataType.DOUBLE,
                 equalTo(result)
             );
         }));
     }
 
     private static void bytesRefs(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.KEYWORD, DataTypes.KEYWORD), () -> {
-            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataTypes.KEYWORD).value());
-            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataTypes.KEYWORD).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD, DataType.KEYWORD), () -> {
+            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataType.KEYWORD).value());
+            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataType.KEYWORD).value());
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.KEYWORD, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.KEYWORD, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.KEYWORD, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.KEYWORD, "field2")
                 ),
                 "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.KEYWORD,
+                DataType.KEYWORD,
                 equalTo(result)
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.TEXT, DataTypes.TEXT), () -> {
-            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataTypes.TEXT).value());
-            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataTypes.TEXT).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT, DataType.TEXT), () -> {
+            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value());
+            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value());
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.TEXT, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.TEXT, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.TEXT, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.TEXT, "field2")
                 ),
                 "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.TEXT,
+                DataType.TEXT,
                 equalTo(result)
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.IP), () -> {
-            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataTypes.IP).value());
-            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataTypes.IP).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.IP), () -> {
+            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataType.IP).value());
+            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataType.IP).value());
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.IP, "field"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.IP, "field")
+                    new TestCaseSupplier.TypedData(field1, DataType.IP, "field"),
+                    new TestCaseSupplier.TypedData(field2, DataType.IP, "field")
                 ),
                 "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.IP,
+                DataType.IP,
                 equalTo(result)
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.VERSION, DataTypes.VERSION), () -> {
-            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataTypes.VERSION).value());
-            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataTypes.VERSION).value());
+        suppliers.add(new TestCaseSupplier(List.of(DataType.VERSION, DataType.VERSION), () -> {
+            List<Object> field1 = randomList(1, 10, () -> randomLiteral(DataType.VERSION).value());
+            List<Object> field2 = randomList(1, 10, () -> randomLiteral(DataType.VERSION).value());
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.VERSION, "field"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.VERSION, "field")
+                    new TestCaseSupplier.TypedData(field1, DataType.VERSION, "field"),
+                    new TestCaseSupplier.TypedData(field2, DataType.VERSION, "field")
                 ),
                 "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.VERSION,
+                DataType.VERSION,
                 equalTo(result)
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.GEO_POINT, DataTypes.GEO_POINT), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.GEO_POINT, DataType.GEO_POINT), () -> {
             List<Object> field1 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint())));
             List<Object> field2 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomPoint())));
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.GEO_POINT, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.GEO_POINT, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.GEO_POINT, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.GEO_POINT, "field2")
                 ),
                 "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.GEO_POINT,
+                DataType.GEO_POINT,
                 equalTo(result)
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.CARTESIAN_POINT, DataTypes.CARTESIAN_POINT), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.CARTESIAN_POINT, DataType.CARTESIAN_POINT), () -> {
             List<Object> field1 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint())));
             List<Object> field2 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomPoint())));
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.CARTESIAN_POINT, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.CARTESIAN_POINT, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.CARTESIAN_POINT, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.CARTESIAN_POINT, "field2")
                 ),
                 "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.CARTESIAN_POINT,
+                DataType.CARTESIAN_POINT,
                 equalTo(result)
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.GEO_SHAPE, DataTypes.GEO_SHAPE), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.GEO_SHAPE, DataType.GEO_SHAPE), () -> {
             List<Object> field1 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))));
             List<Object> field2 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))));
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.GEO_SHAPE, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.GEO_SHAPE, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.GEO_SHAPE, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.GEO_SHAPE, "field2")
                 ),
                 "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.GEO_SHAPE,
+                DataType.GEO_SHAPE,
                 equalTo(result)
             );
         }));
 
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.CARTESIAN_SHAPE, DataTypes.CARTESIAN_SHAPE), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.CARTESIAN_SHAPE, DataType.CARTESIAN_SHAPE), () -> {
             List<Object> field1 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))));
             List<Object> field2 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))));
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.CARTESIAN_SHAPE, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.CARTESIAN_SHAPE, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.CARTESIAN_SHAPE, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.CARTESIAN_SHAPE, "field2")
                 ),
                 "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.CARTESIAN_SHAPE,
+                DataType.CARTESIAN_SHAPE,
                 equalTo(result)
             );
         }));
     }
 
     private static void nulls(List<TestCaseSupplier> suppliers) {
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.INTEGER, DataType.INTEGER), () -> {
             List<Integer> field2 = randomList(2, 10, () -> randomInt());
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(null, DataTypes.INTEGER, "field1"),
-                    new TestCaseSupplier.TypedData(field2, DataTypes.INTEGER, "field2")
+                    new TestCaseSupplier.TypedData(null, DataType.INTEGER, "field1"),
+                    new TestCaseSupplier.TypedData(field2, DataType.INTEGER, "field2")
                 ),
                 "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 equalTo(null)
             );
         }));
-        suppliers.add(new TestCaseSupplier(List.of(DataTypes.INTEGER, DataTypes.INTEGER), () -> {
+        suppliers.add(new TestCaseSupplier(List.of(DataType.INTEGER, DataType.INTEGER), () -> {
             List<Integer> field1 = randomList(2, 10, () -> randomInt());
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(field1, DataTypes.INTEGER, "field1"),
-                    new TestCaseSupplier.TypedData(null, DataTypes.INTEGER, "field2")
+                    new TestCaseSupplier.TypedData(field1, DataType.INTEGER, "field1"),
+                    new TestCaseSupplier.TypedData(null, DataType.INTEGER, "field2")
                 ),
                 "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]",
-                DataTypes.INTEGER,
+                DataType.INTEGER,
                 equalTo(null)
             );
         }));

From db310b462397d7a98ca240a67f76a5371196ab32 Mon Sep 17 00:00:00 2001
From: Oleksandr Kolomiiets <oleksandr.kolomiiets@elastic.co>
Date: Tue, 4 Jun 2024 13:36:20 -0700
Subject: [PATCH 148/208] Enable fallback synthetic source for `point` and
 `shape` (#109312)

---
 docs/changelog/109312.yaml                    |   5 +
 .../index/mapper/PointFieldMapper.java        |   5 +
 .../index/mapper/ShapeFieldMapper.java        |   5 +
 .../mapper/CartesianFieldMapperTests.java     |  12 +-
 ...GeoShapeWithDocValuesFieldMapperTests.java | 145 +-------
 .../GeometricShapeSyntheticSourceSupport.java | 165 +++++++++
 .../index/mapper/PointFieldMapperTests.java   | 170 ++++++++-
 .../index/mapper/ShapeFieldMapperTests.java   |  25 +-
 .../test/spatial/140_synthetic_source.yml     | 325 ++++++++++++++++++
 9 files changed, 707 insertions(+), 150 deletions(-)
 create mode 100644 docs/changelog/109312.yaml
 create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeometricShapeSyntheticSourceSupport.java

diff --git a/docs/changelog/109312.yaml b/docs/changelog/109312.yaml
new file mode 100644
index 0000000000000..594d3f90e8fd1
--- /dev/null
+++ b/docs/changelog/109312.yaml
@@ -0,0 +1,5 @@
+pr: 109312
+summary: Enable fallback synthetic source for `point` and `shape`
+area: Mapping
+type: feature
+issues: []
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java
index 1657a3bf7fbce..2a2a6ad164677 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java
@@ -180,6 +180,11 @@ public FieldMapper.Builder getMergeBuilder() {
         return new Builder(simpleName(), builder.ignoreMalformed.getDefaultValue().value()).init(this);
     }
 
+    @Override
+    protected SyntheticSourceMode syntheticSourceMode() {
+        return SyntheticSourceMode.FALLBACK;
+    }
+
     public static class PointFieldType extends AbstractPointFieldType<CartesianPoint> implements ShapeQueryable {
 
         private final ShapeQueryPointProcessor queryProcessor;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java
index 83e434f829591..4d6a909f96c3f 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java
@@ -249,6 +249,11 @@ public ShapeFieldType fieldType() {
         return (ShapeFieldType) super.fieldType();
     }
 
+    @Override
+    protected SyntheticSourceMode syntheticSourceMode() {
+        return SyntheticSourceMode.FALLBACK;
+    }
+
     public static class CartesianShapeDocValuesField extends AbstractScriptFieldFactory<CartesianShapeValues.CartesianShapeValue>
         implements
             Field<CartesianShapeValues.CartesianShapeValue>,
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java
index b17aa1b175f2b..a2560bb38c6ce 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperTestCase;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin;
@@ -20,6 +21,7 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
@@ -32,7 +34,15 @@ public abstract class CartesianFieldMapperTests extends MapperTestCase {
 
     @Override
     protected Collection<Plugin> getPlugins() {
-        return Collections.singletonList(new LocalStateSpatialPlugin());
+        var plugin = new LocalStateSpatialPlugin();
+        plugin.loadExtensions(new ExtensiblePlugin.ExtensionLoader() {
+            @Override
+            public <T> List<T> loadExtensions(Class<T> extensionPointType) {
+                return List.of();
+            }
+        });
+
+        return Collections.singletonList(plugin);
     }
 
     @Override
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java
index 507fe80291d89..55fcf1b7d39b4 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java
@@ -9,10 +9,7 @@
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.geo.GeoJson;
-import org.elasticsearch.common.geo.GeometryNormalizer;
 import org.elasticsearch.common.geo.Orientation;
-import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.utils.GeometryValidator;
 import org.elasticsearch.geometry.utils.WellKnownBinary;
@@ -30,17 +27,14 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.function.Function;
-import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -434,144 +428,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        // Almost like GeoShapeType but no circles
-        enum ShapeType {
-            POINT,
-            LINESTRING,
-            POLYGON,
-            MULTIPOINT,
-            MULTILINESTRING,
-            MULTIPOLYGON,
-            GEOMETRYCOLLECTION,
-            ENVELOPE
-        }
-
-        return new SyntheticSourceSupport() {
-            @Override
-            public boolean preservesExactSource() {
-                return true;
-            }
-
-            @Override
-            public SyntheticSourceExample example(int maxValues) throws IOException {
-                if (randomBoolean()) {
-                    Value v = generateValue();
-                    if (v.blockLoaderOutput != null) {
-                        return new SyntheticSourceExample(v.input, v.output, v.blockLoaderOutput, this::mapping);
-                    }
-                    return new SyntheticSourceExample(v.input, v.output, this::mapping);
-                }
-
-                List<Value> values = randomList(1, maxValues, this::generateValue);
-                List<Object> in = values.stream().map(Value::input).toList();
-                List<Object> out = values.stream().map(Value::output).toList();
-
-                // Block loader infrastructure will never return nulls
-                List<Object> outBlockList = values.stream()
-                    .filter(v -> v.input != null)
-                    .map(v -> v.blockLoaderOutput != null ? v.blockLoaderOutput : v.output)
-                    .toList();
-                var outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
-
-                return new SyntheticSourceExample(in, out, outBlock, this::mapping);
-            }
-
-            private record Value(Object input, Object output, String blockLoaderOutput) {
-                Value(Object input, Object output) {
-                    this(input, output, null);
-                }
-            }
-
-            private Value generateValue() {
-                if (ignoreMalformed && randomBoolean()) {
-                    List<Supplier<Object>> choices = List.of(
-                        () -> randomAlphaOfLength(3),
-                        ESTestCase::randomInt,
-                        ESTestCase::randomLong,
-                        ESTestCase::randomFloat,
-                        ESTestCase::randomDouble
-                    );
-                    Object v = randomFrom(choices).get();
-                    return new Value(v, v);
-                }
-                if (randomBoolean()) {
-                    return new Value(null, null);
-                }
-
-                var type = randomFrom(ShapeType.values());
-                var isGeoJson = randomBoolean();
-
-                switch (type) {
-                    case POINT -> {
-                        var point = GeometryTestUtils.randomPoint(false);
-                        return value(point, isGeoJson);
-                    }
-                    case LINESTRING -> {
-                        var line = GeometryTestUtils.randomLine(false);
-                        return value(line, isGeoJson);
-                    }
-                    case POLYGON -> {
-                        var polygon = GeometryTestUtils.randomPolygon(false);
-                        return value(polygon, isGeoJson);
-                    }
-                    case MULTIPOINT -> {
-                        var multiPoint = GeometryTestUtils.randomMultiPoint(false);
-                        return value(multiPoint, isGeoJson);
-                    }
-                    case MULTILINESTRING -> {
-                        var multiPoint = GeometryTestUtils.randomMultiLine(false);
-                        return value(multiPoint, isGeoJson);
-                    }
-                    case MULTIPOLYGON -> {
-                        var multiPolygon = GeometryTestUtils.randomMultiPolygon(false);
-                        return value(multiPolygon, isGeoJson);
-                    }
-                    case GEOMETRYCOLLECTION -> {
-                        var multiPolygon = GeometryTestUtils.randomGeometryCollectionWithoutCircle(false);
-                        return value(multiPolygon, isGeoJson);
-                    }
-                    case ENVELOPE -> {
-                        var rectangle = GeometryTestUtils.randomRectangle();
-                        var wktString = WellKnownText.toWKT(rectangle);
-
-                        return new Value(wktString, wktString);
-                    }
-                    default -> throw new UnsupportedOperationException("Unsupported shape");
-                }
-            }
-
-            private static Value value(Geometry geometry, boolean isGeoJson) {
-                var wktString = WellKnownText.toWKT(geometry);
-                var normalizedWktString = GeometryNormalizer.needsNormalize(Orientation.RIGHT, geometry)
-                    ? WellKnownText.toWKT(GeometryNormalizer.apply(Orientation.RIGHT, geometry))
-                    : wktString;
-
-                if (isGeoJson) {
-                    var map = GeoJson.toMap(geometry);
-                    return new Value(map, map, normalizedWktString);
-                }
-
-                return new Value(wktString, wktString, normalizedWktString);
-            }
-
-            private void mapping(XContentBuilder b) throws IOException {
-                b.field("type", "geo_shape");
-                if (rarely()) {
-                    b.field("index", false);
-                }
-                if (rarely()) {
-                    b.field("doc_values", false);
-                }
-                if (ignoreMalformed) {
-                    b.field("ignore_malformed", true);
-                }
-            }
-
-            @Override
-            public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
-                return List.of();
-            }
-        };
+        return new GeometricShapeSyntheticSourceSupport(ignoreMalformed);
     }
 
     @Override
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeometricShapeSyntheticSourceSupport.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeometricShapeSyntheticSourceSupport.java
new file mode 100644
index 0000000000000..4325eb41ceefa
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeometricShapeSyntheticSourceSupport.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.index.mapper;
+
+import org.elasticsearch.common.geo.GeoJson;
+import org.elasticsearch.common.geo.GeometryNormalizer;
+import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.ShapeType;
+import org.elasticsearch.geometry.utils.WellKnownText;
+import org.elasticsearch.index.mapper.MapperTestCase;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.apache.lucene.tests.util.LuceneTestCase.rarely;
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
+import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+import static org.elasticsearch.test.ESTestCase.randomList;
+
+/**
+ * Synthetic source support for fields the index geometry shapes: shape, geo_shape.
+ */
+public class GeometricShapeSyntheticSourceSupport implements MapperTestCase.SyntheticSourceSupport {
+    private final boolean ignoreMalformed;
+
+    public GeometricShapeSyntheticSourceSupport(boolean ignoreMalformed) {
+        this.ignoreMalformed = ignoreMalformed;
+    }
+
+    @Override
+    public boolean preservesExactSource() {
+        return true;
+    }
+
+    @Override
+    public MapperTestCase.SyntheticSourceExample example(int maxValues) throws IOException {
+        if (randomBoolean()) {
+            Value v = generateValue();
+            if (v.blockLoaderOutput != null) {
+                return new MapperTestCase.SyntheticSourceExample(v.input, v.output, v.blockLoaderOutput, this::mapping);
+            }
+            return new MapperTestCase.SyntheticSourceExample(v.input, v.output, this::mapping);
+        }
+
+        List<Value> values = randomList(1, maxValues, this::generateValue);
+        List<Object> in = values.stream().map(Value::input).toList();
+        List<Object> out = values.stream().map(Value::output).toList();
+
+        // Block loader infrastructure will never return nulls
+        List<Object> outBlockList = values.stream()
+            .filter(v -> v.input != null)
+            .map(v -> v.blockLoaderOutput != null ? v.blockLoaderOutput : v.output)
+            .toList();
+        var outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
+
+        return new MapperTestCase.SyntheticSourceExample(in, out, outBlock, this::mapping);
+    }
+
+    private record Value(Object input, Object output, String blockLoaderOutput) {
+        Value(Object input, Object output) {
+            this(input, output, null);
+        }
+    }
+
+    private Value generateValue() {
+        if (ignoreMalformed && randomBoolean()) {
+            List<Supplier<Object>> choices = List.of(
+                () -> randomAlphaOfLength(3),
+                ESTestCase::randomInt,
+                ESTestCase::randomLong,
+                ESTestCase::randomFloat,
+                ESTestCase::randomDouble
+            );
+            Object v = randomFrom(choices).get();
+            return new Value(v, v);
+        }
+        if (randomBoolean()) {
+            return new Value(null, null);
+        }
+
+        var type = randomFrom(ShapeType.values());
+        var isGeoJson = randomBoolean();
+
+        return switch (type) {
+            // LINEARRING and CIRCLE are not supported as inputs to fields so just return points
+            case POINT, LINEARRING, CIRCLE -> {
+                var point = GeometryTestUtils.randomPoint(false);
+                yield value(point, isGeoJson);
+            }
+            case MULTIPOINT -> {
+                var multiPoint = GeometryTestUtils.randomMultiPoint(false);
+                yield value(multiPoint, isGeoJson);
+            }
+            case LINESTRING -> {
+                var line = GeometryTestUtils.randomLine(false);
+                yield value(line, isGeoJson);
+            }
+            case MULTILINESTRING -> {
+                var multiPoint = GeometryTestUtils.randomMultiLine(false);
+                yield value(multiPoint, isGeoJson);
+            }
+            case POLYGON -> {
+                var polygon = GeometryTestUtils.randomPolygon(false);
+                yield value(polygon, isGeoJson);
+            }
+            case MULTIPOLYGON -> {
+                var multiPolygon = GeometryTestUtils.randomMultiPolygon(false);
+                yield value(multiPolygon, isGeoJson);
+            }
+            case GEOMETRYCOLLECTION -> {
+                var multiPolygon = GeometryTestUtils.randomGeometryCollectionWithoutCircle(false);
+                yield value(multiPolygon, isGeoJson);
+            }
+            case ENVELOPE -> {
+                var rectangle = GeometryTestUtils.randomRectangle();
+                var wktString = WellKnownText.toWKT(rectangle);
+
+                yield new Value(wktString, wktString);
+            }
+        };
+    }
+
+    private static Value value(Geometry geometry, boolean isGeoJson) {
+        var wktString = WellKnownText.toWKT(geometry);
+        var normalizedWktString = GeometryNormalizer.needsNormalize(Orientation.RIGHT, geometry)
+            ? WellKnownText.toWKT(GeometryNormalizer.apply(Orientation.RIGHT, geometry))
+            : wktString;
+
+        if (isGeoJson) {
+            var map = GeoJson.toMap(geometry);
+            return new Value(map, map, normalizedWktString);
+        }
+
+        return new Value(wktString, wktString, normalizedWktString);
+    }
+
+    private void mapping(XContentBuilder b) throws IOException {
+        b.field("type", "geo_shape");
+        if (rarely()) {
+            b.field("index", false);
+        }
+        if (rarely()) {
+            b.field("doc_values", false);
+        }
+        if (ignoreMalformed) {
+            b.field("ignore_malformed", true);
+        }
+    }
+
+    @Override
+    public List<MapperTestCase.SyntheticSourceInvalidExample> invalidExample() throws IOException {
+        return List.of();
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java
index b2b250c6d81bd..ad622109e1748 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java
@@ -6,9 +6,17 @@
  */
 package org.elasticsearch.xpack.spatial.index.mapper;
 
+import org.apache.lucene.document.XYDocValuesField;
 import org.apache.lucene.document.XYPointField;
+import org.apache.lucene.geo.GeoEncodingUtils;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.utils.GeometryValidator;
+import org.elasticsearch.geometry.utils.WellKnownBinary;
+import org.elasticsearch.geometry.utils.WellKnownText;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentParsingException;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -22,7 +30,11 @@
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
 
 import static org.elasticsearch.geometry.utils.Geohash.stringEncode;
 import static org.hamcrest.Matchers.containsString;
@@ -419,7 +431,132 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+        return syntheticSourceSupport(ignoreMalformed, false);
+    }
+
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed, boolean columnReader) {
+        return new SyntheticSourceSupport() {
+            private final boolean ignoreZValue = usually();
+            private final CartesianPoint nullValue = usually() ? null : randomCartesianPoint();
+
+            @Override
+            public boolean preservesExactSource() {
+                return true;
+            }
+
+            @Override
+            public SyntheticSourceExample example(int maxVals) {
+                if (randomBoolean()) {
+                    Value v = generateValue();
+
+                    if (v.point == null) {
+                        return new SyntheticSourceExample(v.representation(), v.representation(), null, this::mapping);
+                    } else if (columnReader) {
+                        return new SyntheticSourceExample(v.representation(), v.representation(), encode(v.point()), this::mapping);
+                    }
+                    return new SyntheticSourceExample(v.representation(), v.representation(), v.point().toWKT(), this::mapping);
+                }
+                List<Value> values = randomList(1, maxVals, this::generateValue);
+                var representations = values.stream().map(Value::representation).toList();
+
+                if (columnReader) {
+                    // When reading doc-values, the block is a list of encoded longs
+                    List<Long> outBlockList = values.stream()
+                        .map(Value::point)
+                        .filter(Objects::nonNull)
+                        .map(this::encode)
+                        .sorted()
+                        .toList();
+                    Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
+                    return new SyntheticSourceExample(representations, representations, outBlock, this::mapping);
+                } else {
+                    // When reading row-stride, the block is a list of WKT encoded BytesRefs
+                    List<String> outBlockList = values.stream()
+                        .map(Value::point)
+                        .filter(Objects::nonNull)
+                        .map(CartesianPoint::toWKT)
+                        .toList();
+                    Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
+                    return new SyntheticSourceExample(representations, representations, outBlock, this::mapping);
+                }
+            }
+
+            private record Value(CartesianPoint point, Object representation) {}
+
+            private Value generateValue() {
+                if (nullValue != null && randomBoolean()) {
+                    return new Value(nullValue, null);
+                }
+
+                if (ignoreMalformed) {
+                    // #exampleMalformedValues() covers a lot of cases
+
+                    // nice complex object
+                    return new Value(null, Map.of("one", 1, "two", List.of(2, 22, 222), "three", Map.of("three", 33)));
+                }
+
+                CartesianPoint point = randomCartesianPoint();
+                return new Value(point, randomInputFormat(point));
+            }
+
+            private CartesianPoint randomCartesianPoint() {
+                Point point = GeometryTestUtils.randomPoint(false);
+                return decode(encode(new CartesianPoint(point.getLat(), point.getLon())));
+            }
+
+            private Object randomInputFormat(CartesianPoint point) {
+                return switch (randomInt(4)) {
+                    case 0 -> Map.of("x", point.getX(), "y", point.getY());
+                    case 1 -> new double[] { point.getX(), point.getY() };
+                    case 2 -> "POINT( " + point.getX() + " " + point.getY() + " )";
+                    case 3 -> point.toString();
+                    default -> {
+                        List<Double> coords = new ArrayList<>();
+                        coords.add(point.getX());
+                        coords.add(point.getY());
+                        if (ignoreZValue) {
+                            coords.add(randomDouble());
+                        }
+                        yield Map.of("coordinates", coords, "type", "point");
+                    }
+                };
+            }
+
+            private long encode(CartesianPoint point) {
+                return new XYDocValuesField("f", (float) point.getX(), (float) point.getY()).numericValue().longValue();
+            }
+
+            private CartesianPoint decode(long point) {
+                double lat = GeoEncodingUtils.decodeLatitude((int) (point >> 32));
+                double lon = GeoEncodingUtils.decodeLongitude((int) (point & 0xFFFFFFFF));
+                return new CartesianPoint(lat, lon);
+            }
+
+            private void mapping(XContentBuilder b) throws IOException {
+                b.field("type", "point");
+                if (ignoreZValue == false || rarely()) {
+                    b.field("ignore_z_value", ignoreZValue);
+                }
+                if (nullValue != null) {
+                    b.field("null_value", randomInputFormat(nullValue));
+                }
+                if (rarely()) {
+                    b.field("index", false);
+                }
+                if (rarely()) {
+                    b.field("store", false);
+                }
+                if (ignoreMalformed) {
+                    b.field("ignore_malformed", true);
+                }
+            }
+
+            @Override
+            public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
+                return List.of();
+            }
+        };
     }
 
     @Override
@@ -427,11 +564,36 @@ protected IngestScriptSupport ingestScriptSupport() {
         throw new AssumptionViolatedException("not supported");
     }
 
+    @Override
+    protected Function<Object, Object> loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) {
+        if (columnReader) {
+            // When using column reader, we expect the output to be doc-values (which means encoded longs)
+            return v -> asJacksonNumberOutput(((Number) v).longValue());
+        } else {
+            // When using row-stride reader, we expect the output to be WKT encoded BytesRef
+            return v -> asWKT((BytesRef) v);
+        }
+    }
+
+    protected static Object asJacksonNumberOutput(long l) {
+        // Cast to int to mimic jackson-core behaviour in NumberOutput.outputLong()
+        // that is called when deserializing expected value in SyntheticSourceExample.
+        if (l < 0 && l >= Integer.MIN_VALUE || l >= 0 && l <= Integer.MAX_VALUE) {
+            return (int) l;
+        } else {
+            return l;
+        }
+    }
+
+    protected static Object asWKT(BytesRef value) {
+        // Internally we use WKB in BytesRef, but for test assertions we want to use WKT for readability
+        Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, value.bytes);
+        return WellKnownText.toWKT(geometry);
+    }
+
     @Override
     protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
-        // TODO: Support testing both reading from source as well as reading from doc-values
         MappedFieldType ft = mapper.fieldType(loaderFieldName);
-        PointFieldMapper.PointFieldType point = (PointFieldMapper.PointFieldType) ft;
-        return new BlockReaderSupport(point.isIndexed() == false && ft.hasDocValues(), false, mapper, loaderFieldName);
+        return new BlockReaderSupport(ft.hasDocValues(), false, mapper, loaderFieldName);
     }
 }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java
index 26d349a7ee5a6..28297f32297e6 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java
@@ -8,8 +8,13 @@
 
 import org.apache.lucene.document.ShapeField;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.utils.GeometryValidator;
+import org.elasticsearch.geometry.utils.WellKnownBinary;
+import org.elasticsearch.geometry.utils.WellKnownText;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper;
@@ -29,6 +34,7 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
 
 import static org.elasticsearch.geometry.utils.Geohash.stringEncode;
 import static org.hamcrest.Matchers.containsString;
@@ -362,7 +368,24 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+        return new GeometricShapeSyntheticSourceSupport(ignoreMalformed);
+    }
+
+    @Override
+    protected Function<Object, Object> loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) {
+        return v -> asWKT((BytesRef) v);
+    }
+
+    protected static Object asWKT(BytesRef value) {
+        // Internally we use WKB in BytesRef, but for test assertions we want to use WKT for readability
+        Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, value.bytes);
+        return WellKnownText.toWKT(geometry);
+    }
+
+    @Override
+    protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
+        // Synthetic source is currently not supported.
+        return new BlockReaderSupport(false, false, mapper, loaderFieldName);
     }
 
     @Override
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml
index 569c310c3b244..ccc6cd8627b53 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml
@@ -1,5 +1,9 @@
 ---
 "geo_shape":
+  - requires:
+      cluster_features: ["mapper.source.synthetic_source_fallback"]
+      reason: introduced in 8.15.0
+
   - do:
       indices.create:
         index: test
@@ -62,6 +66,10 @@
 
 ---
 "geo_shape with ignore_malformed":
+  - requires:
+      cluster_features: ["mapper.source.synthetic_source_fallback"]
+      reason: introduced in 8.15.0
+
   - do:
       indices.create:
         index: test
@@ -139,8 +147,161 @@
 
   - match: { _source.shape: ["POINT (-77.03653 1000)", "POINT (-71.34 41.12)"] }
 
+---
+"shape":
+  - requires:
+      cluster_features: ["mapper.source.synthetic_source_fallback"]
+      reason: introduced in 8.15.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              shape:
+                type: shape
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          shape:
+            type: "Polygon"
+            coordinates: [[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]], [[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]]
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          shape: "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0), (100.2 0.2, 100.8 0.2, 100.8 0.8, 100.2 0.8, 100.2 0.2))"
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          shape: ["POINT (-77.03653 38.897676)", {"type" : "LineString", "coordinates" : [[-77.03653, 38.897676], [-77.009051, 38.889939]]}]
+
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      get:
+        index: test
+        id: "1"
+
+  - match: { _source.shape.type: "Polygon" }
+  - match: { _source.shape.coordinates: [[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]], [[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]] }
+
+  - do:
+      get:
+        index: test
+        id: "2"
+
+  - match: { _source.shape: "POLYGON ((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0), (100.2 0.2, 100.8 0.2, 100.8 0.8, 100.2 0.8, 100.2 0.2))" }
+
+  - do:
+      get:
+        index: test
+        id: "3"
+
+  - match: { _source.shape: ["POINT (-77.03653 38.897676)", {"type" : "LineString", "coordinates" : [[-77.03653, 38.897676], [-77.009051, 38.889939]]}] }
+
+---
+"shape with ignore_malformed":
+  - requires:
+      cluster_features: ["mapper.source.synthetic_source_fallback"]
+      reason: introduced in 8.15.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              shape:
+                type: shape
+                ignore_malformed: true
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          shape: 500
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          shape:
+            string: "string"
+            array: [{ "a": 1 }, { "b": 2 }]
+            object: { "foo": "bar" }
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          shape: ["POINT (-77.03653 38.897676)", "potato", "POINT (-71.34 41.12)"]
+
+  - do:
+      index:
+        index: test
+        id: "4"
+        body:
+          shape: ["POINT (-77.03653 1000)", "POINT (-71.34 41.12)"]
+
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      get:
+        index: test
+        id: "1"
+
+  - match: { _source.shape: 500 }
+
+  - do:
+      get:
+        index: test
+        id: "2"
+
+  - match: { _source.shape.string: "string" }
+  - match: { _source.shape.array: [{ "a": 1 }, { "b": 2 }] }
+  - match: { _source.shape.object: { "foo": "bar" } }
+
+  - do:
+      get:
+        index: test
+        id: "3"
+
+  - match: { _source.shape: ["POINT (-77.03653 38.897676)", "potato", "POINT (-71.34 41.12)"] }
+
+  - do:
+      get:
+        index: test
+        id: "4"
+
+  - match: { _source.shape: ["POINT (-77.03653 1000)", "POINT (-71.34 41.12)"] }
+
 ---
 "geo_point":
+  - requires:
+      cluster_features: ["gte_v8.3.0"]
+      reason: introduced in 8.3.0
+
   - do:
       indices.create:
         index: test
@@ -249,3 +410,167 @@
 
   - match: { _source.point.lon: -71.34000029414892 }
   - match: { _source.point.lat: 41.119999922811985 }
+
+---
+"point":
+  - requires:
+      cluster_features: ["mapper.source.synthetic_source_fallback"]
+      reason: introduced in 8.15.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              point:
+                type: point
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          point:
+            type: "Point"
+            coordinates: [-71.34, 41.12]
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          point: "POINT (-71.34 41.12)"
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          point:
+            x: -71.34
+            y: 41.12
+
+  - do:
+      index:
+        index: test
+        id: "4"
+        body:
+          point: [ -71.34, 41.12 ]
+
+  - do:
+      index:
+        index: test
+        id: "5"
+        body:
+          point: "41.12,-71.34"
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      get:
+        index: test
+        id: "1"
+
+  - match: { _source.point.type: "Point" }
+  - match: { _source.point.coordinates: [-71.34, 41.12] }
+
+  - do:
+      get:
+        index: test
+        id: "2"
+
+  - match: { _source.point: "POINT (-71.34 41.12)" }
+
+  - do:
+      get:
+        index: test
+        id: "3"
+
+  - match: { _source.point.x: -71.34 }
+  - match: { _source.point.y: 41.12 }
+
+  - do:
+      get:
+        index: test
+        id: "4"
+
+  - match: { _source.point: [ -71.34, 41.12 ] }
+
+  - do:
+      get:
+        index: test
+        id: "5"
+
+  - match: { _source.point: "41.12,-71.34" }
+
+---
+"point with ignore_malformed":
+  - requires:
+      cluster_features: ["mapper.source.synthetic_source_fallback"]
+      reason: introduced in 8.15.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              point:
+                type: point
+                ignore_malformed: true
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          point:
+            string: "string"
+            array: [{ "a": 1 }, { "b": 2 }]
+            object: { "foo": "bar" }
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          point: ["POINT (-77.03653 38.897676)", "potato", "POINT (-71.34 41.12)"]
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          point: ["POINT (-77.03653 1000)", "POINT (-71.34 41.12)"]
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      get:
+        index: test
+        id: "1"
+
+  - match: { _source.point.string: "string" }
+  - match: { _source.point.array: [{ "a": 1 }, { "b": 2 }] }
+  - match: { _source.point.object: { "foo": "bar" } }
+
+  - do:
+      get:
+        index: test
+        id: "2"
+
+  - match: { _source.point: ["POINT (-77.03653 38.897676)", "potato", "POINT (-71.34 41.12)"] }
+
+  - do:
+      get:
+        index: test
+        id: "3"
+
+  - match: { _source.point: ["POINT (-77.03653 1000)", "POINT (-71.34 41.12)"] }

From bb3ff8e9248866cc1a872a3b42b7a094f5ce16f9 Mon Sep 17 00:00:00 2001
From: Parker Timmins <parker.timmins@elastic.co>
Date: Tue, 4 Jun 2024 15:32:43 -0600
Subject: [PATCH 149/208] ESQL: add REPEAT string function (#109220)

Add support for the string manipulation function REPEAT(string, number). This function concatenates the string argument with itself the specified number of times. If number is 0 an empty string is returned. If number is less than 0, null is returned and a warning is logged. If number is less than 0 and is a constant, the query will fail without executing.
---
 docs/changelog/109220.yaml                    |   5 +
 .../functions/description/repeat.asciidoc     |   5 +
 .../esql/functions/examples/repeat.asciidoc   |  13 ++
 .../functions/kibana/definition/repeat.json   |  47 ++++++
 .../esql/functions/kibana/docs/repeat.md      |  11 ++
 .../esql/functions/layout/repeat.asciidoc     |  15 ++
 .../esql/functions/parameters/repeat.asciidoc |   9 +
 .../esql/functions/signature/repeat.svg       |   1 +
 .../esql/functions/string-functions.asciidoc  |   2 +
 .../esql/functions/types/repeat.asciidoc      |  10 ++
 .../src/main/resources/meta.csv-spec          |   6 +-
 .../src/main/resources/string.csv-spec        | 105 ++++++++++++
 .../string/RepeatConstantEvaluator.java       | 136 +++++++++++++++
 .../scalar/string/RepeatEvaluator.java        | 157 ++++++++++++++++++
 .../xpack/esql/action/EsqlCapabilities.java   |   6 +
 .../function/EsqlFunctionRegistry.java        |   4 +-
 .../function/scalar/string/Repeat.java        | 148 +++++++++++++++++
 .../xpack/esql/io/stream/PlanNamedTypes.java  |  14 ++
 .../scalar/string/RepeatStaticTests.java      | 101 +++++++++++
 .../function/scalar/string/RepeatTests.java   | 118 +++++++++++++
 20 files changed, 911 insertions(+), 2 deletions(-)
 create mode 100644 docs/changelog/109220.yaml
 create mode 100644 docs/reference/esql/functions/description/repeat.asciidoc
 create mode 100644 docs/reference/esql/functions/examples/repeat.asciidoc
 create mode 100644 docs/reference/esql/functions/kibana/definition/repeat.json
 create mode 100644 docs/reference/esql/functions/kibana/docs/repeat.md
 create mode 100644 docs/reference/esql/functions/layout/repeat.asciidoc
 create mode 100644 docs/reference/esql/functions/parameters/repeat.asciidoc
 create mode 100644 docs/reference/esql/functions/signature/repeat.svg
 create mode 100644 docs/reference/esql/functions/types/repeat.asciidoc
 create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatConstantEvaluator.java
 create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatEvaluator.java
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatStaticTests.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java

diff --git a/docs/changelog/109220.yaml b/docs/changelog/109220.yaml
new file mode 100644
index 0000000000000..b8efa8f784d7a
--- /dev/null
+++ b/docs/changelog/109220.yaml
@@ -0,0 +1,5 @@
+pr: 109220
+summary: "ESQL: add REPEAT string function"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/reference/esql/functions/description/repeat.asciidoc b/docs/reference/esql/functions/description/repeat.asciidoc
new file mode 100644
index 0000000000000..e008eca90e9e4
--- /dev/null
+++ b/docs/reference/esql/functions/description/repeat.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Returns a string constructed by concatenating `string` with itself the specified `number` of times.
diff --git a/docs/reference/esql/functions/examples/repeat.asciidoc b/docs/reference/esql/functions/examples/repeat.asciidoc
new file mode 100644
index 0000000000000..97bede2517f10
--- /dev/null
+++ b/docs/reference/esql/functions/examples/repeat.asciidoc
@@ -0,0 +1,13 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Example*
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/string.csv-spec[tag=repeat]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/string.csv-spec[tag=repeat-result]
+|===
+
diff --git a/docs/reference/esql/functions/kibana/definition/repeat.json b/docs/reference/esql/functions/kibana/definition/repeat.json
new file mode 100644
index 0000000000000..2ac94e12c1796
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/definition/repeat.json
@@ -0,0 +1,47 @@
+{
+  "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
+  "type" : "eval",
+  "name" : "repeat",
+  "description" : "Returns a string constructed by concatenating `string` with itself the specified `number` of times.",
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "string",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "String expression."
+        },
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Number times to repeat."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string",
+          "type" : "text",
+          "optional" : false,
+          "description" : "String expression."
+        },
+        {
+          "name" : "number",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Number times to repeat."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    }
+  ],
+  "examples" : [
+    "ROW a = \"Hello!\"\n| EVAL triple_a = REPEAT(a, 3);"
+  ]
+}
diff --git a/docs/reference/esql/functions/kibana/docs/repeat.md b/docs/reference/esql/functions/kibana/docs/repeat.md
new file mode 100644
index 0000000000000..cc46e8282d9fe
--- /dev/null
+++ b/docs/reference/esql/functions/kibana/docs/repeat.md
@@ -0,0 +1,11 @@
+<!--
+This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+-->
+
+### REPEAT
+Returns a string constructed by concatenating `string` with itself the specified `number` of times.
+
+```
+ROW a = "Hello!"
+| EVAL triple_a = REPEAT(a, 3);
+```
diff --git a/docs/reference/esql/functions/layout/repeat.asciidoc b/docs/reference/esql/functions/layout/repeat.asciidoc
new file mode 100644
index 0000000000000..c001b22260485
--- /dev/null
+++ b/docs/reference/esql/functions/layout/repeat.asciidoc
@@ -0,0 +1,15 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-repeat]]
+=== `REPEAT`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/repeat.svg[Embedded,opts=inline]
+
+include::../parameters/repeat.asciidoc[]
+include::../description/repeat.asciidoc[]
+include::../types/repeat.asciidoc[]
+include::../examples/repeat.asciidoc[]
diff --git a/docs/reference/esql/functions/parameters/repeat.asciidoc b/docs/reference/esql/functions/parameters/repeat.asciidoc
new file mode 100644
index 0000000000000..263191340f5d9
--- /dev/null
+++ b/docs/reference/esql/functions/parameters/repeat.asciidoc
@@ -0,0 +1,9 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`string`::
+String expression.
+
+`number`::
+Number times to repeat.
diff --git a/docs/reference/esql/functions/signature/repeat.svg b/docs/reference/esql/functions/signature/repeat.svg
new file mode 100644
index 0000000000000..591c20d567845
--- /dev/null
+++ b/docs/reference/esql/functions/signature/repeat.svg
@@ -0,0 +1 @@
+<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="432" height="46" viewbox="0 0 432 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .k{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .s{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m92 0h10m32 0h10m92 0h10m32 0h10m92 0h10m32 0h5"/><rect class="s" x="5" y="5" width="92" height="36"/><text class="k" x="15" y="31">REPEAT</text><rect class="s" x="107" y="5" width="32" height="36" rx="7"/><text class="syn" x="117" y="31">(</text><rect class="s" x="149" y="5" width="92" height="36" rx="7"/><text class="k" x="159" y="31">string</text><rect class="s" x="251" y="5" width="32" height="36" rx="7"/><text class="syn" x="261" y="31">,</text><rect class="s" x="293" y="5" width="92" height="36" rx="7"/><text class="k" x="303" y="31">number</text><rect class="s" x="395" y="5" width="32" height="36" rx="7"/><text class="syn" x="405" y="31">)</text></svg>
\ No newline at end of file
diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc
index bfe7f1f0fecd8..d4b120ad1c45b 100644
--- a/docs/reference/esql/functions/string-functions.asciidoc
+++ b/docs/reference/esql/functions/string-functions.asciidoc
@@ -15,6 +15,7 @@
 * <<esql-length>>
 * <<esql-locate>>
 * <<esql-ltrim>>
+* <<esql-repeat>>
 * <<esql-replace>>
 * <<esql-right>>
 * <<esql-rtrim>>
@@ -34,6 +35,7 @@ include::layout/left.asciidoc[]
 include::layout/length.asciidoc[]
 include::layout/locate.asciidoc[]
 include::layout/ltrim.asciidoc[]
+include::layout/repeat.asciidoc[]
 include::layout/replace.asciidoc[]
 include::layout/right.asciidoc[]
 include::layout/rtrim.asciidoc[]
diff --git a/docs/reference/esql/functions/types/repeat.asciidoc b/docs/reference/esql/functions/types/repeat.asciidoc
new file mode 100644
index 0000000000000..49e4e80094d7b
--- /dev/null
+++ b/docs/reference/esql/functions/types/repeat.asciidoc
@@ -0,0 +1,10 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+string | number | result
+keyword | integer | keyword
+text | integer | keyword
+|===
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
index 621944ce26e0b..4ab10f78c3128 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
@@ -60,6 +60,7 @@ date now()
 "double|integer|long percentile(number:double|integer|long, percentile:double|integer|long)"
 double pi()
 "double pow(base:double|integer|long|unsigned_long, exponent:double|integer|long|unsigned_long)"
+"keyword repeat(string:keyword|text, number:integer)"
 "keyword replace(string:keyword|text, regex:keyword|text, newString:keyword|text)"
 "keyword right(string:keyword|text, length:integer)"
 "double|integer|long|unsigned_long round(number:double|integer|long|unsigned_long, ?decimals:integer)"
@@ -176,6 +177,7 @@ now           |null                                |null
 percentile    |[number, percentile]                |["double|integer|long", "double|integer|long"]                                                                                    |[, ]
 pi            |null                                |null                                                                                                                              |null
 pow           |[base, exponent]                    |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"]                                                        |["Numeric expression for the base. If `null`\, the function returns `null`.", "Numeric expression for the exponent. If `null`\, the function returns `null`."]
+repeat        |[string, number]                    |["keyword|text", integer]                                                                                                         |[String expression., Number times to repeat.]
 replace       |[string, regex, newString]          |["keyword|text", "keyword|text", "keyword|text"]                                                                                  |[String expression., Regular expression., Replacement string.]
 right         |[string, length]                    |["keyword|text", integer]                                                                                                         |[The string from which to returns a substring., The number of characters to return.]
 round         |[number, decimals]                  |["double|integer|long|unsigned_long", integer]                                                                                    |["The numeric value to round. If `null`\, the function returns `null`.", "The number of decimal places to round to. Defaults to 0. If `null`\, the function returns `null`."]
@@ -293,6 +295,7 @@ now           |Returns current date and time.
 percentile    |The value at which a certain percentage of observed values occur.
 pi            |Returns {wikipedia}/Pi[Pi], the ratio of a circle's circumference to its diameter.
 pow           |Returns the value of `base` raised to the power of `exponent`.
+repeat        |Returns a string constructed by concatenating `string` with itself the specified `number` of times.
 replace       |The function substitutes in the string `str` any match of the regular expression `regex` with the replacement string `newStr`.
 right         |Return the substring that extracts 'length' chars from 'str' starting from the right.
 round         |Rounds a number to the specified number of decimal places. Defaults to 0, which returns the nearest integer. If the precision is a negative number, rounds to the number of digits left of the decimal point.
@@ -411,6 +414,7 @@ now           |date
 percentile    |"double|integer|long"                                                                                                       |[false, false]              |false           |true
 pi            |double                                                                                                                      |null                        |false           |false
 pow           |double                                                                                                                      |[false, false]              |false           |false
+repeat        |keyword                                                                                                                     |[false, false]              |false           |false
 replace       |keyword                                                                                                                     |[false, false, false]       |false           |false
 right         |keyword                                                                                                                     |[false, false]              |false           |false
 round         |"double|integer|long|unsigned_long"                                                                                         |[false, true]               |false           |false
@@ -479,5 +483,5 @@ countFunctions#[skip:-8.14.99, reason:BIN added]
 meta functions |  stats  a = count(*), b = count(*), c = count(*) |  mv_expand c;
 
 a:long | b:long | c:long
-108    | 108    | 108
+109    | 109    | 109
 ;
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
index 3cb7c6ef0f594..063b74584a28b 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
@@ -1454,3 +1454,108 @@ emp_no:integer | first_name:keyword | e:keyword    | d:keyword
 10030          | null               | null         | null
 10031          | null               | null         | null
 ;
+
+repeat
+required_capability: repeat 
+// tag::repeat[]
+ROW a = "Hello!"
+| EVAL triple_a = REPEAT(a, 3);
+// end::repeat[]
+
+// tag::repeat-result[]
+a:keyword | triple_a:keyword
+Hello! | Hello!Hello!Hello!
+// end::repeat-result[]
+;
+
+repeatUtf16Emoji
+required_capability: repeat 
+row a = "🐱Meow!🐶Woof!" | eval repeated = repeat(a, 2);
+
+a:keyword | repeated:keyword
+🐱Meow!🐶Woof! | 🐱Meow!🐶Woof!🐱Meow!🐶Woof!
+;
+
+repeatLength
+required_capability: repeat 
+row a = "cheese" | eval repeated_len = length(repeat(a, 5));
+
+a:keyword | repeated_len:integer
+cheese | 30
+;
+
+repeatSubstring
+required_capability: repeat 
+row a = "catcat" | eval repeated = repeat(substring(a, 4), 2);
+
+a:keyword | repeated:keyword
+catcat | catcat
+;
+
+repeatZero
+required_capability: repeat 
+row a = "cheese" | eval repeated = repeat(a, 0);
+
+a:keyword | repeated:keyword
+cheese | ""
+;
+
+repeatNegative
+required_capability: repeat 
+
+from employees | sort emp_no | limit 1 | eval repeated = repeat(first_name, emp_no - 10002) | keep first_name, repeated;
+
+warning:Line 1:58: evaluation of [repeat(first_name, emp_no - 10002)] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 1:58: java.lang.IllegalArgumentException: Number parameter cannot be negative, found [-1]
+
+first_name:keyword | repeated:keyword
+Georgi | null
+;
+
+repeatVariableNumber
+required_capability: repeat 
+from employees 
+| sort emp_no 
+| limit 4 
+| eval first_repeated = repeat(first_name, emp_no - 10001) 
+| keep emp_no, first_name, first_repeated;
+
+emp_no:integer | first_name:keyword  | first_repeated:keyword
+10001 | Georgi    | ""
+10002 | Bezalel   | Bezalel   
+10003 | Parto     | PartoParto 
+10004 | Chirstian | ChirstianChirstianChirstian   
+;
+
+repeatStringNull
+required_capability: repeat 
+row n = 2 | eval repeated = repeat(null, n);
+
+n:integer | repeated:keyword
+2 | null
+;
+
+repeatNumberNull
+required_capability: repeat 
+row s = "cheese" | eval repeated = repeat(s, null);
+
+s:keyword | repeated:keyword
+cheese    | null
+;
+
+repeatBothArgsFromIndex
+required_capability: repeat 
+FROM employees 
+| EVAL first_name=REPEAT(first_name, languages) 
+| KEEP emp_no, languages, first_name
+| WHERE emp_no < 10005 
+| SORT emp_no;
+
+emp_no:integer | languages:integer | first_name:keyword
+10001 | 2 | GeorgiGeorgi
+10002 | 5 | BezalelBezalelBezalelBezalelBezalel
+10003 | 4 | PartoPartoPartoParto
+10004 | 5 | ChirstianChirstianChirstianChirstianChirstian
+;
+
+
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatConstantEvaluator.java
new file mode 100644
index 0000000000000..e83c7c7720828
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatConstantEvaluator.java
@@ -0,0 +1,136 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import java.util.function.Function;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Repeat}.
+ * This class is generated. Do not edit it.
+ */
+public final class RepeatConstantEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final BreakingBytesRefBuilder scratch;
+
+  private final EvalOperator.ExpressionEvaluator str;
+
+  private final int number;
+
+  private final DriverContext driverContext;
+
+  public RepeatConstantEvaluator(Source source, BreakingBytesRefBuilder scratch,
+      EvalOperator.ExpressionEvaluator str, int number, DriverContext driverContext) {
+    this.scratch = scratch;
+    this.str = str;
+    this.number = number;
+    this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) {
+      BytesRefVector strVector = strBlock.asVector();
+      if (strVector == null) {
+        return eval(page.getPositionCount(), strBlock);
+      }
+      return eval(page.getPositionCount(), strVector);
+    }
+  }
+
+  public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock) {
+    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      BytesRef strScratch = new BytesRef();
+      position: for (int p = 0; p < positionCount; p++) {
+        if (strBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (strBlock.getValueCount(p) != 1) {
+          if (strBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        try {
+          result.appendBytesRef(Repeat.processConstantNumber(scratch, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), number));
+        } catch (IllegalArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  public BytesRefBlock eval(int positionCount, BytesRefVector strVector) {
+    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      BytesRef strScratch = new BytesRef();
+      position: for (int p = 0; p < positionCount; p++) {
+        try {
+          result.appendBytesRef(Repeat.processConstantNumber(scratch, strVector.getBytesRef(p, strScratch), number));
+        } catch (IllegalArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "RepeatConstantEvaluator[" + "str=" + str + ", number=" + number + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(scratch, str);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final Function<DriverContext, BreakingBytesRefBuilder> scratch;
+
+    private final EvalOperator.ExpressionEvaluator.Factory str;
+
+    private final int number;
+
+    public Factory(Source source, Function<DriverContext, BreakingBytesRefBuilder> scratch,
+        EvalOperator.ExpressionEvaluator.Factory str, int number) {
+      this.source = source;
+      this.scratch = scratch;
+      this.str = str;
+      this.number = number;
+    }
+
+    @Override
+    public RepeatConstantEvaluator get(DriverContext context) {
+      return new RepeatConstantEvaluator(source, scratch.apply(context), str.get(context), number, context);
+    }
+
+    @Override
+    public String toString() {
+      return "RepeatConstantEvaluator[" + "str=" + str + ", number=" + number + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatEvaluator.java
new file mode 100644
index 0000000000000..3723a35283c4b
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatEvaluator.java
@@ -0,0 +1,157 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License
+// 2.0; you may not use this file except in compliance with the Elastic License
+// 2.0.
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import java.lang.IllegalArgumentException;
+import java.lang.Override;
+import java.lang.String;
+import java.util.function.Function;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Warnings;
+
+/**
+ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Repeat}.
+ * This class is generated. Do not edit it.
+ */
+public final class RepeatEvaluator implements EvalOperator.ExpressionEvaluator {
+  private final Warnings warnings;
+
+  private final BreakingBytesRefBuilder scratch;
+
+  private final EvalOperator.ExpressionEvaluator str;
+
+  private final EvalOperator.ExpressionEvaluator number;
+
+  private final DriverContext driverContext;
+
+  public RepeatEvaluator(Source source, BreakingBytesRefBuilder scratch,
+      EvalOperator.ExpressionEvaluator str, EvalOperator.ExpressionEvaluator number,
+      DriverContext driverContext) {
+    this.scratch = scratch;
+    this.str = str;
+    this.number = number;
+    this.driverContext = driverContext;
+    this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source);
+  }
+
+  @Override
+  public Block eval(Page page) {
+    try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) {
+      try (IntBlock numberBlock = (IntBlock) number.eval(page)) {
+        BytesRefVector strVector = strBlock.asVector();
+        if (strVector == null) {
+          return eval(page.getPositionCount(), strBlock, numberBlock);
+        }
+        IntVector numberVector = numberBlock.asVector();
+        if (numberVector == null) {
+          return eval(page.getPositionCount(), strBlock, numberBlock);
+        }
+        return eval(page.getPositionCount(), strVector, numberVector);
+      }
+    }
+  }
+
+  public BytesRefBlock eval(int positionCount, BytesRefBlock strBlock, IntBlock numberBlock) {
+    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      BytesRef strScratch = new BytesRef();
+      position: for (int p = 0; p < positionCount; p++) {
+        if (strBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (strBlock.getValueCount(p) != 1) {
+          if (strBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        if (numberBlock.isNull(p)) {
+          result.appendNull();
+          continue position;
+        }
+        if (numberBlock.getValueCount(p) != 1) {
+          if (numberBlock.getValueCount(p) > 1) {
+            warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value"));
+          }
+          result.appendNull();
+          continue position;
+        }
+        try {
+          result.appendBytesRef(Repeat.process(scratch, strBlock.getBytesRef(strBlock.getFirstValueIndex(p), strScratch), numberBlock.getInt(numberBlock.getFirstValueIndex(p))));
+        } catch (IllegalArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  public BytesRefBlock eval(int positionCount, BytesRefVector strVector, IntVector numberVector) {
+    try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+      BytesRef strScratch = new BytesRef();
+      position: for (int p = 0; p < positionCount; p++) {
+        try {
+          result.appendBytesRef(Repeat.process(scratch, strVector.getBytesRef(p, strScratch), numberVector.getInt(p)));
+        } catch (IllegalArgumentException e) {
+          warnings.registerException(e);
+          result.appendNull();
+        }
+      }
+      return result.build();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "RepeatEvaluator[" + "str=" + str + ", number=" + number + "]";
+  }
+
+  @Override
+  public void close() {
+    Releasables.closeExpectNoException(scratch, str, number);
+  }
+
+  static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+    private final Source source;
+
+    private final Function<DriverContext, BreakingBytesRefBuilder> scratch;
+
+    private final EvalOperator.ExpressionEvaluator.Factory str;
+
+    private final EvalOperator.ExpressionEvaluator.Factory number;
+
+    public Factory(Source source, Function<DriverContext, BreakingBytesRefBuilder> scratch,
+        EvalOperator.ExpressionEvaluator.Factory str,
+        EvalOperator.ExpressionEvaluator.Factory number) {
+      this.source = source;
+      this.scratch = scratch;
+      this.str = str;
+      this.number = number;
+    }
+
+    @Override
+    public RepeatEvaluator get(DriverContext context) {
+      return new RepeatEvaluator(source, scratch.apply(context), str.get(context), number.get(context), context);
+    }
+
+    @Override
+    public String toString() {
+      return "RepeatEvaluator[" + "str=" + str + ", number=" + number + "]";
+    }
+  }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index 907422b179657..13ad47850cd01 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -57,6 +57,11 @@ public class EsqlCapabilities {
      */
     private static final String LOOKUP = "lookup";
 
+    /**
+     * Support for requesting the "REPEAT" command.
+     */
+    private static final String REPEAT = "repeat";
+
     public static final Set<String> CAPABILITIES = capabilities();
 
     /**
@@ -72,6 +77,7 @@ private static Set<String> capabilities() {
         caps.add(ST_CENTROID_AGG_OPTIMIZED);
         caps.add(METADATA_IGNORED_FIELD);
         caps.add(FN_MV_APPEND);
+        caps.add(REPEAT);
 
         if (Build.current().isSnapshot()) {
             caps.add(LOOKUP);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index aa335a1d09400..8fd6ebe8d7d69 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -103,6 +103,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split;
@@ -234,7 +235,8 @@ private FunctionDefinition[][] functions() {
                 def(EndsWith.class, EndsWith::new, "ends_with"),
                 def(ToLower.class, ToLower::new, "to_lower"),
                 def(ToUpper.class, ToUpper::new, "to_upper"),
-                def(Locate.class, Locate::new, "locate") },
+                def(Locate.class, Locate::new, "locate"),
+                def(Repeat.class, Repeat::new, "repeat") },
             // date
             new FunctionDefinition[] {
                 def(DateDiff.class, DateDiff::new, "date_diff"),
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java
new file mode 100644
index 0000000000000..e8ad0a83829fe
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.ann.Evaluator;
+import org.elasticsearch.compute.ann.Fixed;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.function.OptionalArgument;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+
+import static org.elasticsearch.common.unit.ByteSizeUnit.MB;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
+
+public class Repeat extends EsqlScalarFunction implements OptionalArgument {
+
+    static final long MAX_REPEATED_LENGTH = MB.toBytes(1);
+
+    private final Expression str;
+    private final Expression number;
+
+    @FunctionInfo(
+        returnType = "keyword",
+        description = "Returns a string constructed by concatenating `string` with itself the specified `number` of times.",
+        examples = @Example(file = "string", tag = "repeat")
+    )
+    public Repeat(
+        Source source,
+        @Param(name = "string", type = { "keyword", "text" }, description = "String expression.") Expression str,
+        @Param(name = "number", type = { "integer" }, description = "Number times to repeat.") Expression number
+    ) {
+        super(source, Arrays.asList(str, number));
+        this.str = str;
+        this.number = number;
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataType.KEYWORD;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution resolution = isString(str, sourceText(), FIRST);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        return isType(number, dt -> dt == DataType.INTEGER, sourceText(), SECOND, "integer");
+    }
+
+    @Override
+    public boolean foldable() {
+        return str.foldable() && number.foldable();
+    }
+
+    @Evaluator(extraName = "Constant", warnExceptions = { IllegalArgumentException.class })
+    static BytesRef processConstantNumber(
+        @Fixed(includeInToString = false, build = true) BreakingBytesRefBuilder scratch,
+        BytesRef str,
+        @Fixed int number
+    ) {
+        return processInner(scratch, str, number);
+    }
+
+    @Evaluator(warnExceptions = { IllegalArgumentException.class })
+    static BytesRef process(@Fixed(includeInToString = false, build = true) BreakingBytesRefBuilder scratch, BytesRef str, int number) {
+        if (number < 0) {
+            throw new IllegalArgumentException("Number parameter cannot be negative, found [" + number + "]");
+        }
+        return processInner(scratch, str, number);
+    }
+
+    static BytesRef processInner(BreakingBytesRefBuilder scratch, BytesRef str, int number) {
+        int repeatedLen = str.length * number;
+        if (repeatedLen > MAX_REPEATED_LENGTH) {
+            throw new IllegalArgumentException(
+                "Creating repeated strings with more than [" + MAX_REPEATED_LENGTH + "] bytes is not supported"
+            );
+        }
+        scratch.grow(repeatedLen);
+        scratch.clear();
+        for (int i = 0; i < number; ++i) {
+            scratch.append(str);
+        }
+        return scratch.bytesRefView();
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new Repeat(source(), newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Repeat::new, str, number);
+    }
+
+    @Override
+    public ExpressionEvaluator.Factory toEvaluator(Function<Expression, ExpressionEvaluator.Factory> toEvaluator) {
+        ExpressionEvaluator.Factory strExpr = toEvaluator.apply(str);
+
+        if (number.foldable()) {
+            int num = (int) number.fold();
+            if (num < 0) {
+                throw new IllegalArgumentException("Number parameter cannot be negative, found [" + number + "]");
+            }
+            return new RepeatConstantEvaluator.Factory(
+                source(),
+                context -> new BreakingBytesRefBuilder(context.breaker(), "repeat"),
+                strExpr,
+                num
+            );
+        }
+
+        ExpressionEvaluator.Factory numberExpr = toEvaluator.apply(number);
+        return new RepeatEvaluator.Factory(
+            source(),
+            context -> new BreakingBytesRefBuilder(context.breaker(), "repeat"),
+            strExpr,
+            numberExpr
+        );
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index bad07bea9926b..fe2c704743f5c 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -148,6 +148,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split;
@@ -399,6 +400,7 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(ScalarFunction.class, Substring.class, PlanNamedTypes::writeSubstring, PlanNamedTypes::readSubstring),
             of(ScalarFunction.class, Locate.class, PlanNamedTypes::writeLocate, PlanNamedTypes::readLocate),
             of(ScalarFunction.class, Left.class, PlanNamedTypes::writeLeft, PlanNamedTypes::readLeft),
+            of(ScalarFunction.class, Repeat.class, PlanNamedTypes::writeRepeat, PlanNamedTypes::readRepeat),
             of(ScalarFunction.class, Right.class, PlanNamedTypes::writeRight, PlanNamedTypes::readRight),
             of(ScalarFunction.class, Split.class, PlanNamedTypes::writeSplit, PlanNamedTypes::readSplit),
             of(ScalarFunction.class, Tau.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar),
@@ -1554,6 +1556,18 @@ static void writeLeft(PlanStreamOutput out, Left left) throws IOException {
         out.writeExpression(fields.get(1));
     }
 
+    static Repeat readRepeat(PlanStreamInput in) throws IOException {
+        return new Repeat(Source.readFrom(in), in.readExpression(), in.readExpression());
+    }
+
+    static void writeRepeat(PlanStreamOutput out, Repeat repeat) throws IOException {
+        repeat.source().writeTo(out);
+        List<Expression> fields = repeat.children();
+        assert fields.size() == 2;
+        out.writeExpression(fields.get(0));
+        out.writeExpression(fields.get(1));
+    }
+
     static Right readRight(PlanStreamInput in) throws IOException {
         return new Right(Source.readFrom(in), in.readExpression(), in.readExpression());
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatStaticTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatStaticTests.java
new file mode 100644
index 0000000000000..dc266066bd424
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatStaticTests.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockFactory;
+import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.esql.TestBlockFactory;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
+import org.junit.After;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * These tests create rows that are 1MB in size. Test classes
+ * which extend AbstractFunctionTestCase rerun test cases with
+ * many randomized inputs. Unfortunately, tests are run with
+ * limited memory, and instantiating many copies of these
+ * tests with large rows causes out of memory.
+ */
+public class RepeatStaticTests extends ESTestCase {
+
+    public void testAlmostTooBig() {
+        String str = randomAlphaOfLength(1);
+        int number = (int) Repeat.MAX_REPEATED_LENGTH;
+        String repeated = process(str, number);
+        assertThat(repeated, equalTo(str.repeat(number)));
+    }
+
+    public void testTooBig() {
+        String str = randomAlphaOfLength(1);
+        int number = (int) Repeat.MAX_REPEATED_LENGTH + 1;
+        String repeated = process(str, number);
+        assertNull(repeated);
+        assertWarnings(
+            "Line -1:-1: java.lang.IllegalArgumentException: Creating repeated strings with more than [1048576] bytes is not supported",
+            "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded."
+        );
+    }
+
+    public String process(String str, int number) {
+        try (
+            var eval = AbstractFunctionTestCase.evaluator(
+                new Repeat(Source.EMPTY, field("string", DataType.KEYWORD), field("number", DataType.INTEGER))
+            ).get(driverContext());
+            Block block = eval.eval(row(List.of(new BytesRef(str), number)));
+        ) {
+            return block.isNull(0) ? null : ((BytesRef) BlockUtils.toJavaObject(block, 0)).utf8ToString();
+        }
+    }
+
+    /**
+     * The following fields and methods were borrowed from AbstractFunctionTestCase
+     */
+    private final List<CircuitBreaker> breakers = Collections.synchronizedList(new ArrayList<>());
+
+    private static Page row(List<Object> values) {
+        return new Page(1, BlockUtils.fromListRow(TestBlockFactory.getNonBreakingInstance(), values));
+    }
+
+    private static FieldAttribute field(String name, DataType type) {
+        return new FieldAttribute(Source.synthetic(name), name, new EsField(name, type, Map.of(), true));
+    }
+
+    private DriverContext driverContext() {
+        BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofMb(256)).withCircuitBreaking();
+        CircuitBreaker breaker = bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST);
+        breakers.add(breaker);
+        return new DriverContext(bigArrays, new BlockFactory(breaker, bigArrays));
+    }
+
+    @After
+    public void allMemoryReleased() {
+        for (CircuitBreaker breaker : breakers) {
+            assertThat(breaker.getUsed(), equalTo(0L));
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java
new file mode 100644
index 0000000000000..cb89dc168b928
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class RepeatTests extends AbstractFunctionTestCase {
+    public RepeatTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+
+        List<TestCaseSupplier> cases = new ArrayList<>();
+
+        cases.add(new TestCaseSupplier("Repeat basic test", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
+            String text = randomAlphaOfLength(10);
+            int number = between(0, 10);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(number, DataType.INTEGER, "number")
+                ),
+                "RepeatEvaluator[str=Attribute[channel=0], number=Attribute[channel=1]]",
+                DataType.KEYWORD,
+                equalTo(new BytesRef(text.repeat(number)))
+            );
+        }));
+
+        cases.add(new TestCaseSupplier("Repeat basic test with text input", List.of(DataType.TEXT, DataType.INTEGER), () -> {
+            String text = randomAlphaOfLength(10);
+            int number = between(0, 10);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.TEXT, "str"),
+                    new TestCaseSupplier.TypedData(number, DataType.INTEGER, "number")
+                ),
+                "RepeatEvaluator[str=Attribute[channel=0], number=Attribute[channel=1]]",
+                DataType.KEYWORD,
+                equalTo(new BytesRef(text.repeat(number)))
+            );
+        }));
+
+        cases.add(new TestCaseSupplier("Repeat with number zero", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
+            String text = randomAlphaOfLength(10);
+            int number = 0;
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(number, DataType.INTEGER, "number")
+                ),
+                "RepeatEvaluator[str=Attribute[channel=0], number=Attribute[channel=1]]",
+                DataType.KEYWORD,
+                equalTo(new BytesRef(""))
+            );
+        }));
+
+        cases.add(new TestCaseSupplier("Repeat Unicode", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
+            String text = randomUnicodeOfLength(10);
+            int number = randomIntBetween(0, 10);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(number, DataType.INTEGER, "number")
+                ),
+                "RepeatEvaluator[str=Attribute[channel=0], number=Attribute[channel=1]]",
+                DataType.KEYWORD,
+                equalTo(new BytesRef(text.repeat(number)))
+            );
+        }));
+
+        cases.add(new TestCaseSupplier("Repeat Negative Number", List.of(DataType.KEYWORD, DataType.INTEGER), () -> {
+            String text = randomAlphaOfLength(10);
+            int number = randomIntBetween(-10, -1);
+            return new TestCaseSupplier.TestCase(
+                List.of(
+                    new TestCaseSupplier.TypedData(new BytesRef(text), DataType.KEYWORD, "str"),
+                    new TestCaseSupplier.TypedData(number, DataType.INTEGER, "number")
+                ),
+                "RepeatEvaluator[str=Attribute[channel=0], number=Attribute[channel=1]]",
+                DataType.KEYWORD,
+                nullValue()
+            ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
+                .withWarning("Line -1:-1: java.lang.IllegalArgumentException: Number parameter cannot be negative, found [" + number + "]")
+                .withFoldingException(IllegalArgumentException.class, "Number parameter cannot be negative, found [" + number + "]");
+        }));
+
+        cases = anyNullIsNull(true, cases);
+        cases = errorsForCasesWithoutExamples(cases);
+        return parameterSuppliersFromTypedData(cases);
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Repeat(source, args.get(0), args.get(1));
+    }
+}

From bb4b4e0a4ecb7b9ebb52852b2cdbb072163a492a Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Tue, 4 Jun 2024 18:06:42 -0400
Subject: [PATCH 150/208] Unmute mixed cluster test client yaml tests (#109367)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 muted-tests.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 147267cb1d197..f26a9edaff72d 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -56,10 +56,6 @@ tests:
 - class: "org.elasticsearch.client.RestClientSingleHostIntegTests"
   issue: "https://github.com/elastic/elasticsearch/issues/102717"
   method: "testRequestResetAndAbort"
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/109362
-- class: org.elasticsearch.upgrades.UpgradeClusterClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/109364
 
 # Examples:
 #

From aa12cbe144e084fd6b405b8f7ee220c7f0642248 Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Tue, 4 Jun 2024 19:07:09 -0400
Subject: [PATCH 151/208] Unmute AggsIT (#109372)

should be fixed by: https://github.com/elastic/elasticsearch/pull/109349

closes: https://github.com/elastic/elasticsearch/issues/109322
---
 muted-tests.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index f26a9edaff72d..45e7a3802688b 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -47,9 +47,6 @@ tests:
 - class: "org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109318"
   method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)}"
-- class: "org.elasticsearch.upgrades.AggregationsIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109322"
-  method: "testTerms"
 - class: "org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT"
   issue: "https://github.com/elastic/elasticsearch/issues/101598"
   method: "testFeatureImportanceValues"

From f161a1e89009262f693273fced31e5bfdf2435c3 Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Tue, 4 Jun 2024 19:17:13 -0400
Subject: [PATCH 152/208] Muting test for #109373 (#109377)

related: https://github.com/elastic/elasticsearch/issues/109373
---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index 45e7a3802688b..3a11e4729a670 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -53,6 +53,9 @@ tests:
 - class: "org.elasticsearch.client.RestClientSingleHostIntegTests"
   issue: "https://github.com/elastic/elasticsearch/issues/102717"
   method: "testRequestResetAndAbort"
+- class: org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppendTests
+  issue: https://github.com/elastic/elasticsearch/issues/109373
+  method: testEvaluateBlockWithoutNulls
 
 # Examples:
 #

From 67378f6b65da5fd3350dd3cbcc2e5f0c0e5d904e Mon Sep 17 00:00:00 2001
From: Yang Wang <yang.wang@elastic.co>
Date: Wed, 5 Jun 2024 09:40:20 +1000
Subject: [PATCH 153/208] [Test] Better error message for test failure of
 testBatchCloseIndices (#109278)

The test failed because the indices failed to close. Unfortunately, the
AssertionError in clusterStateListner prevents the updateTask from being
completed which in turn swallows the actual underlying exception that
made the indices fail to close. This PR captures the closed indices in
the clusterStateListener and asserts them at the end the test so that
actual exception will be reported when calling future.actionGet when the
test fails again.

Relates: #109187
---
 ...etadataIndexStateServiceBatchingTests.java | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceBatchingTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceBatchingTests.java
index 1cb5650d26930..a093178c04814 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceBatchingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceBatchingTests.java
@@ -20,6 +20,9 @@
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.TimeUnit;
@@ -98,8 +101,11 @@ public void testBatchCloseIndices() throws Exception {
         createIndex("test-3", indicesAdmin().prepareCreate("test-3"));
         ensureGreen("test-1", "test-2", "test-3");
 
-        final var assertingListener = closedIndexCountListener(3);
-        clusterService.addListener(assertingListener);
+        final List<String[]> observedClosedIndices = Collections.synchronizedList(new ArrayList<>());
+        final ClusterStateListener closedIndicesStateListener = event -> observedClosedIndices.add(
+            event.state().metadata().getConcreteAllClosedIndices()
+        );
+        clusterService.addListener(closedIndicesStateListener);
 
         final var block1 = blockMasterService(masterService);
         block1.run(); // wait for block
@@ -123,12 +129,12 @@ public void testBatchCloseIndices() throws Exception {
         block2.run(); // release block
 
         // assert that the requests were acknowledged
-        final var resp1 = future1.get();
+        final var resp1 = safeGet(future1);
         assertAcked(resp1);
         assertThat(resp1.getIndices(), hasSize(1));
         assertThat(resp1.getIndices().get(0).getIndex().getName(), is("test-1"));
 
-        final var resp2 = future2.get();
+        final var resp2 = safeGet(future2);
         assertAcked(resp2);
         assertThat(resp2.getIndices(), hasSize(2));
         assertThat(resp2.getIndices().stream().map(r -> r.getIndex().getName()).toList(), containsInAnyOrder("test-2", "test-3"));
@@ -139,7 +145,10 @@ public void testBatchCloseIndices() throws Exception {
             assertThat(indexMetadata.getState(), is(State.CLOSE));
         }
 
-        clusterService.removeListener(assertingListener);
+        clusterService.removeListener(closedIndicesStateListener);
+        observedClosedIndices.forEach(
+            indices -> assertThat("unexpected closed indices: " + Arrays.toString(indices), indices.length, oneOf(0, 3))
+        );
     }
 
     public void testBatchBlockIndices() throws Exception {

From a51a9f75c0a49c0209a294a1672a9955f9a89db8 Mon Sep 17 00:00:00 2001
From: Yang Wang <yang.wang@elastic.co>
Date: Wed, 5 Jun 2024 10:00:13 +1000
Subject: [PATCH 154/208] [Test] Use UnsafePlainActionFuture for
 addRentionLease response (#109330)

When a transport action fails, it can complete the listener without
using the specified executor (#107439). I believe this is what happend
in the test failure #109163. The remote is removed before the CCR
metaddata is removed. A concurrent add retention lease action can
encounter exception which is then completed inline with the same generic
thread pool.

Resolves: #109163 Relates: #107439
---
 muted-tests.yml                                               | 3 ---
 .../java/org/elasticsearch/xpack/ccr/CcrRetentionLeases.java  | 4 +++-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 3a11e4729a670..87e2eb3530b38 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -38,9 +38,6 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/esql/esql-async-query-api/line_17}
   issue: https://github.com/elastic/elasticsearch/issues/109260
-- class: org.elasticsearch.xpack.ccr.LocalIndexFollowingIT
-  method: testRemoveRemoteConnection
-  issue: https://github.com/elastic/elasticsearch/issues/109163
 - class: "org.elasticsearch.index.engine.frozen.FrozenIndexIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109315"
   method: "testTimestampFieldTypeExposedByAllIndicesServices"
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrRetentionLeases.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrRetentionLeases.java
index cdb5bf67b4712..e1d1d84a4eca5 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrRetentionLeases.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrRetentionLeases.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.UnsafePlainActionFuture;
 import org.elasticsearch.client.internal.RemoteClusterClient;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.core.TimeValue;
@@ -18,6 +19,7 @@
 import org.elasticsearch.index.seqno.RetentionLeaseAlreadyExistsException;
 import org.elasticsearch.index.seqno.RetentionLeaseNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Locale;
 import java.util.Optional;
@@ -79,7 +81,7 @@ public static Optional<RetentionLeaseAlreadyExistsException> syncAddRetentionLea
         final TimeValue timeout
     ) {
         try {
-            final PlainActionFuture<ActionResponse.Empty> response = new PlainActionFuture<>();
+            final PlainActionFuture<ActionResponse.Empty> response = new UnsafePlainActionFuture<>(ThreadPool.Names.GENERIC);
             asyncAddRetentionLease(leaderShardId, retentionLeaseId, retainingSequenceNumber, remoteClient, response);
             response.actionGet(timeout);
             return Optional.empty();

From c5892a7044b51012e5680346c3bf7e03a70ce70d Mon Sep 17 00:00:00 2001
From: Moritz Mack <moritz@mackmail.net>
Date: Wed, 5 Jun 2024 06:37:05 +0200
Subject: [PATCH 155/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/108628

---
 muted-tests.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 87e2eb3530b38..be4845ccec9e9 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -19,7 +19,8 @@ tests:
   method: "testGuessIsDayFirstFromLocale"
 - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108857"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
+    \ dependent mappings / dates}"
 - class: "org.elasticsearch.upgrades.SearchStatesIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108991"
   method: "testCanMatch"
@@ -28,7 +29,8 @@ tests:
   method: "testTrainedModelInference"
 - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109188"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
+    \ dependent mappings / dates}"
 - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109189"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
@@ -43,7 +45,8 @@ tests:
   method: "testTimestampFieldTypeExposedByAllIndicesServices"
 - class: "org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109318"
-  method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)}"
+  method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling\
+    \ (too complex pattern)}"
 - class: "org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT"
   issue: "https://github.com/elastic/elasticsearch/issues/101598"
   method: "testFeatureImportanceValues"
@@ -53,6 +56,9 @@ tests:
 - class: org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppendTests
   issue: https://github.com/elastic/elasticsearch/issues/109373
   method: testEvaluateBlockWithoutNulls
+- class: "org.elasticsearch.xpack.deprecation.DeprecationHttpIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/108628"
+  method: "testDeprecatedSettingsReturnWarnings"
 
 # Examples:
 #

From 8bc84b6e371f4d34ee8a595c22f463aa1d55b812 Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Wed, 5 Jun 2024 11:14:12 +0200
Subject: [PATCH 156/208] Make XContentType.xContent() a getter (#109264)

Seen this come up in some profiling, wasting some cycles. If we do a
method per type here instead of a getter + field, we pay for a
megamorphic callsite potentially. It's faster and uses less code
anyway to just use a field + getter here.
---
 .../elasticsearch/xcontent/XContentType.java  | 65 +++++--------------
 1 file changed, 15 insertions(+), 50 deletions(-)

diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentType.java
index 242da6fd705dd..71392aeff542b 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentType.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentType.java
@@ -24,7 +24,7 @@ public enum XContentType implements MediaType {
     /**
      * A JSON based content type.
      */
-    JSON(0) {
+    JSON(0, JsonXContent.jsonXContent) {
         @Override
         public String mediaTypeWithoutParameters() {
             return "application/json";
@@ -40,11 +40,6 @@ public String queryParameter() {
             return "json";
         }
 
-        @Override
-        public XContent xContent() {
-            return JsonXContent.jsonXContent;
-        }
-
         @Override
         public Set<HeaderValue> headerValues() {
             return Set.of(new HeaderValue("application/json"), new HeaderValue("application/x-ndjson"), new HeaderValue("application/*"));
@@ -53,7 +48,7 @@ public Set<HeaderValue> headerValues() {
     /**
      * The jackson based smile binary format. Fast and compact binary format.
      */
-    SMILE(1) {
+    SMILE(1, SmileXContent.smileXContent) {
         @Override
         public String mediaTypeWithoutParameters() {
             return "application/smile";
@@ -64,11 +59,6 @@ public String queryParameter() {
             return "smile";
         }
 
-        @Override
-        public XContent xContent() {
-            return SmileXContent.smileXContent;
-        }
-
         @Override
         public Set<HeaderValue> headerValues() {
             return Set.of(new HeaderValue("application/smile"));
@@ -77,7 +67,7 @@ public Set<HeaderValue> headerValues() {
     /**
      * A YAML based content type.
      */
-    YAML(2) {
+    YAML(2, YamlXContent.yamlXContent) {
         @Override
         public String mediaTypeWithoutParameters() {
             return "application/yaml";
@@ -88,11 +78,6 @@ public String queryParameter() {
             return "yaml";
         }
 
-        @Override
-        public XContent xContent() {
-            return YamlXContent.yamlXContent;
-        }
-
         @Override
         public Set<HeaderValue> headerValues() {
             return Set.of(new HeaderValue("application/yaml"));
@@ -101,7 +86,7 @@ public Set<HeaderValue> headerValues() {
     /**
      * A CBOR based content type.
      */
-    CBOR(3) {
+    CBOR(3, CborXContent.cborXContent) {
         @Override
         public String mediaTypeWithoutParameters() {
             return "application/cbor";
@@ -112,11 +97,6 @@ public String queryParameter() {
             return "cbor";
         }
 
-        @Override
-        public XContent xContent() {
-            return CborXContent.cborXContent;
-        }
-
         @Override
         public Set<HeaderValue> headerValues() {
             return Set.of(new HeaderValue("application/cbor"));
@@ -125,7 +105,7 @@ public Set<HeaderValue> headerValues() {
     /**
      * A versioned JSON based content type.
      */
-    VND_JSON(4) {
+    VND_JSON(4, JsonXContent.jsonXContent) {
         @Override
         public String mediaTypeWithoutParameters() {
             return VENDOR_APPLICATION_PREFIX + "json";
@@ -136,11 +116,6 @@ public String queryParameter() {
             return "vnd_json";
         }
 
-        @Override
-        public XContent xContent() {
-            return JsonXContent.jsonXContent;
-        }
-
         @Override
         public Set<HeaderValue> headerValues() {
             return Set.of(
@@ -157,7 +132,7 @@ public XContentType canonical() {
     /**
      * Versioned jackson based smile binary format. Fast and compact binary format.
      */
-    VND_SMILE(5) {
+    VND_SMILE(5, SmileXContent.smileXContent) {
         @Override
         public String mediaTypeWithoutParameters() {
             return VENDOR_APPLICATION_PREFIX + "smile";
@@ -168,11 +143,6 @@ public String queryParameter() {
             return "vnd_smile";
         }
 
-        @Override
-        public XContent xContent() {
-            return SmileXContent.smileXContent;
-        }
-
         @Override
         public Set<HeaderValue> headerValues() {
             return Set.of(new HeaderValue(VENDOR_APPLICATION_PREFIX + "smile", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN)));
@@ -186,7 +156,7 @@ public XContentType canonical() {
     /**
      * A Versioned YAML based content type.
      */
-    VND_YAML(6) {
+    VND_YAML(6, YamlXContent.yamlXContent) {
         @Override
         public String mediaTypeWithoutParameters() {
             return VENDOR_APPLICATION_PREFIX + "yaml";
@@ -197,11 +167,6 @@ public String queryParameter() {
             return "vnd_yaml";
         }
 
-        @Override
-        public XContent xContent() {
-            return YamlXContent.yamlXContent;
-        }
-
         @Override
         public Set<HeaderValue> headerValues() {
             return Set.of(new HeaderValue(VENDOR_APPLICATION_PREFIX + "yaml", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN)));
@@ -215,7 +180,7 @@ public XContentType canonical() {
     /**
      * A Versioned CBOR based content type.
      */
-    VND_CBOR(7) {
+    VND_CBOR(7, CborXContent.cborXContent) {
         @Override
         public String mediaTypeWithoutParameters() {
             return VENDOR_APPLICATION_PREFIX + "cbor";
@@ -226,11 +191,6 @@ public String queryParameter() {
             return "vnd_cbor";
         }
 
-        @Override
-        public XContent xContent() {
-            return CborXContent.cborXContent;
-        }
-
         @Override
         public Set<HeaderValue> headerValues() {
             return Set.of(new HeaderValue(VENDOR_APPLICATION_PREFIX + "cbor", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN)));
@@ -275,8 +235,11 @@ public static XContentType fromMediaType(String mediaTypeHeaderValue) throws Ill
 
     private final int index;
 
-    XContentType(int index) {
+    private final XContent xContent;
+
+    XContentType(int index, XContent xContent) {
         this.index = index;
+        this.xContent = xContent;
     }
 
     public static Byte parseVersion(String mediaType) {
@@ -296,7 +259,9 @@ public String mediaType() {
         return mediaTypeWithoutParameters();
     }
 
-    public abstract XContent xContent();
+    public final XContent xContent() {
+        return xContent;
+    }
 
     public abstract String mediaTypeWithoutParameters();
 

From 7226100458f62ddfb2244bc6ec2b3b605f71d92e Mon Sep 17 00:00:00 2001
From: Rene Groeschke <rene@elastic.co>
Date: Wed, 5 Jun 2024 11:23:04 +0200
Subject: [PATCH 157/208] Make :distribution:docker:check Gradle configuration
 cache compatible (#109083)

fix cache compatibility issue in TestFixturesPlugin and support
yamlRestTest in `:distribution:docker:yamlRestTest`
---
 .../internal/docker/DockerSupportService.java | 30 +++++++
 .../testfixtures/TestFixturesPlugin.java      | 55 +++++++------
 distribution/docker/build.gradle              | 80 +++++++++++--------
 3 files changed, 103 insertions(+), 62 deletions(-)

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java
index 1ec6f023eb565..fd75df6c06b84 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java
@@ -7,6 +7,8 @@
  */
 package org.elasticsearch.gradle.internal.docker;
 
+import com.avast.gradle.dockercompose.ServiceInfo;
+
 import org.elasticsearch.gradle.Architecture;
 import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.Version;
@@ -56,6 +58,9 @@ public abstract class DockerSupportService implements BuildService<DockerSupport
 
     private final ProviderFactory providerFactory;
     private DockerAvailability dockerAvailability;
+    private Map<String, ServiceInfo> serviceInfos;
+    private Map<String, Map<Integer, Integer>> tcpPorts;
+    private Map<String, Map<Integer, Integer>> udpPorts;
 
     @Inject
     public DockerSupportService(ProviderFactory providerFactory) {
@@ -145,6 +150,10 @@ public DockerAvailability getDockerAvailability() {
         return this.dockerAvailability;
     }
 
+    public boolean isArchitectureSupported(Architecture architecture) {
+        return getDockerAvailability().supportedArchitectures().contains(architecture);
+    }
+
     private DockerResult runCommand(List args, DockerValueSource.OutputFilter outputFilter) {
         return providerFactory.of(DockerValueSource.class, params -> {
             params.getParameters().getArgs().addAll(args);
@@ -329,6 +338,27 @@ private void throwDockerRequiredException(final String message, Exception e) {
         );
     }
 
+    public void storeInfo(Map<String, ServiceInfo> servicesInfos) {
+        tcpPorts = servicesInfos.entrySet()
+            .stream()
+            .collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().getTcpPorts()));
+        udpPorts = servicesInfos.entrySet()
+            .stream()
+            .collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().getUdpPorts()));
+    }
+
+    public Map<String, Map<Integer, Integer>> getTcpPorts() {
+        return tcpPorts;
+    }
+
+    public Map<String, Map<Integer, Integer>> getUdpPorts() {
+        return udpPorts;
+    }
+
+    public void setServiceInfos(Map<String, ServiceInfo> serviceInfos) {
+        this.serviceInfos = serviceInfos;
+    }
+
     /**
      * An immutable class that represents the results of a Docker search from {@link #getDockerAvailability()}}.
      */
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java
index 4c5f2abb9515c..a16057220ce89 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java
@@ -9,7 +9,6 @@
 
 import com.avast.gradle.dockercompose.ComposeExtension;
 import com.avast.gradle.dockercompose.DockerComposePlugin;
-import com.avast.gradle.dockercompose.ServiceInfo;
 import com.avast.gradle.dockercompose.tasks.ComposeBuild;
 import com.avast.gradle.dockercompose.tasks.ComposeDown;
 import com.avast.gradle.dockercompose.tasks.ComposePull;
@@ -106,12 +105,6 @@ public void apply(Project project) {
             .register("postProcessFixture", TestFixtureTask.class, task -> {
                 task.getFixturesDir().set(testFixturesDir);
                 task.dependsOn(buildFixture);
-                configureServiceInfoForTask(
-                    task,
-                    project,
-                    false,
-                    (name, port) -> task.getExtensions().getByType(ExtraPropertiesExtension.class).set(name, port)
-                );
             });
 
         maybeSkipTask(dockerSupport, preProcessFixture);
@@ -131,12 +124,20 @@ public void apply(Project project) {
             return composePath != null ? composePath : "/usr/bin/docker-compose";
         }));
 
-        tasks.named("composeUp").configure(t -> {
+        tasks.withType(ComposeUp.class).named("composeUp").configure(t -> {
             // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions
             if (BuildParams.isCi()) {
                 t.usesService(dockerComposeThrottle);
+                t.usesService(dockerSupport);
             }
             t.mustRunAfter(preProcessFixture);
+            t.doLast(new Action<Task>() {
+                @Override
+                public void execute(Task task) {
+                    dockerSupport.get().storeInfo(t.getServicesInfos());
+                }
+            });
+
         });
         tasks.named("composePull").configure(t -> t.mustRunAfter(preProcessFixture));
         tasks.named("composeDown").configure(t -> t.doLast(t2 -> getFileSystemOperations().delete(d -> d.delete(testFixturesDir))));
@@ -153,14 +154,9 @@ public void apply(Project project) {
         tasks.withType(Test.class).configureEach(testTask -> {
             testTask.dependsOn(postProcessFixture);
             testTask.finalizedBy(tasks.named("composeDown"));
-            configureServiceInfoForTask(
-                testTask,
-                project,
-                true,
-                (name, host) -> testTask.getExtensions()
-                    .getByType(SystemPropertyCommandLineArgumentProvider.class)
-                    .systemProperty(name, host)
-            );
+            SystemPropertyCommandLineArgumentProvider sysArgumentsProvider = testTask.getExtensions()
+                .getByType(SystemPropertyCommandLineArgumentProvider.class);
+            configureServiceInfoForTask(testTask, dockerSupport, (name, host) -> sysArgumentsProvider.systemProperty(name, host));
         });
     }
 
@@ -184,31 +180,34 @@ private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, Task ta
 
     private void configureServiceInfoForTask(
         Task task,
-        Project fixtureProject,
-        boolean enableFilter,
+        Provider<DockerSupportService> dockerSupportServiceProvider,
         BiConsumer<String, Integer> consumer
     ) {
         // Configure ports for the tests as system properties.
         // We only know these at execution time so we need to do it in doFirst
+        task.usesService(dockerSupportServiceProvider);
         task.doFirst(new Action<Task>() {
             @Override
             public void execute(Task theTask) {
-                fixtureProject.getExtensions().getByType(ComposeExtension.class).getServicesInfos().entrySet().stream().forEach(entry -> {
+                dockerSupportServiceProvider.get().getTcpPorts().entrySet().stream().forEach(entry -> {
                     String service = entry.getKey();
-                    ServiceInfo infos = entry.getValue();
-                    infos.getTcpPorts().forEach((container, host) -> {
-                        String name = "test.fixtures." + service + ".tcp." + container;
-                        theTask.getLogger().info("port mapping property: {}={}", name, host);
-                        consumer.accept(name, host);
+                    entry.getValue().entrySet().stream().forEach(portMapping -> {
+                        String name = "test.fixtures." + service + ".tcp." + portMapping.getKey();
+                        theTask.getLogger().info("port mapping property: {}={}", name, portMapping.getValue());
+                        consumer.accept(name, portMapping.getValue());
                     });
-                    infos.getUdpPorts().forEach((container, host) -> {
-                        String name = "test.fixtures." + service + ".udp." + container;
-                        theTask.getLogger().info("port mapping property: {}={}", name, host);
-                        consumer.accept(name, host);
+                });
+                dockerSupportServiceProvider.get().getUdpPorts().entrySet().stream().forEach(entry -> {
+                    String service = entry.getKey();
+                    entry.getValue().entrySet().stream().forEach(portMapping -> {
+                        String name = "test.fixtures." + service + ".udp." + portMapping.getKey();
+                        theTask.getLogger().info("port mapping property: {}={}", name, portMapping.getValue());
+                        consumer.accept(name, portMapping.getValue());
                     });
                 });
             }
         });
+
     }
 
     @SuppressWarnings("unchecked")
diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle
index 68ff2028b92a3..85e66ccba34b1 100644
--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@@ -10,7 +10,6 @@ import org.elasticsearch.gradle.internal.docker.ShellRetry
 import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.util.GradleUtils
-
 import java.nio.file.Path
 import java.time.temporal.ChronoUnit
 
@@ -22,6 +21,8 @@ apply plugin: 'elasticsearch.dra-artifacts'
 String buildId = providers.systemProperty('build.id').getOrNull()
 boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false && useDra == false
 
+
+
 repositories {
   // Define a repository that allows Gradle to fetch a resource from GitHub. This
   // is only used to fetch the `tini` binary, when building the Iron Bank docker image
@@ -120,7 +121,7 @@ ext.expansions = { Architecture architecture, DockerBase base ->
   // the image. When developing the Docker images, it's very tedious to completely rebuild
   // an image for every single change. Therefore, outside of CI, we fix the
   // build time to midnight so that the Docker build cache is usable.
-  def buildDate = BuildParams.isCi() ? BuildParams.buildDate : BuildParams.buildDate.truncatedTo(ChronoUnit.DAYS)
+  def buildDate = BuildParams.isCi() ? BuildParams.buildDate : BuildParams.buildDate.truncatedTo(ChronoUnit.DAYS).toString()
 
   return [
     'arch'               : architecture.classifier,
@@ -188,26 +189,29 @@ ext.dockerBuildContext = { Architecture architecture, DockerBase base ->
     }
   }
 }
-
-def createAndSetWritable(Object... locations) {
-  locations.each { location ->
-    File file = file(location)
-    file.mkdirs()
-    file.setWritable(true, false)
-  }
-}
+//
+//def createAndSetWritable(Object... locations) {
+//  locations.each { location ->
+//    File file = file(location)
+//    file.mkdirs()
+//    file.setWritable(true, false)
+//  }
+//}
 
 tasks.register("copyNodeKeyMaterial", Sync) {
+  def certsDir = file("build/certs")
+  def pemFile = file("build/certs/testnode.pem")
+  def crtFile = file("build/certs/testnode.crt")
   from project(':x-pack:plugin:core')
     .files(
       'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
       'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt'
     )
-  into "${buildDir}/certs"
+  into "build/certs"
   doLast {
-    file("${buildDir}/certs").setReadable(true, false)
-    file("${buildDir}/certs/testnode.pem").setReadable(true, false)
-    file("${buildDir}/certs/testnode.crt").setReadable(true, false)
+    certsDir.setReadable(true, false)
+    pemFile.setReadable(true, false)
+    crtFile.setReadable(true, false)
   }
 }
 
@@ -222,19 +226,27 @@ elasticsearch_distributions {
   }
 }
 
+interface Injected {
+  @Inject FileSystemOperations getFs()
+}
+
 tasks.named("preProcessFixture").configure {
   dependsOn elasticsearch_distributions.matching { it.architecture == Architecture.current() }
   dependsOn "copyNodeKeyMaterial"
+  def injected = project.objects.newInstance(Injected)
+  def testFixturesFolder = project.testFixturesDir.absoluteFile
   doLast {
     // tests expect to have an empty repo
-    project.delete(
-      "${testFixturesDir}/repo",
-    )
-    createAndSetWritable(
-      "${testFixturesDir}/repo",
-      "${testFixturesDir}/logs/default-1",
-      "${testFixturesDir}/logs/default-2",
-    )
+    injected.fs.delete {
+      it.delete("${testFixturesFolder}/repo")
+    }
+    ["${testFixturesFolder}/repo",
+     "${testFixturesFolder}/logs/default-1",
+     "${testFixturesFolder}/logs/default-2"].each { location ->
+      File file = new File(location)
+      file.mkdirs()
+      file.setWritable(true, false)
+    }
   }
 }
 
@@ -299,8 +311,8 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         // For some reason, the artifact name can differ depending on what repository we used.
         rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
       }
-
-      onlyIf("$architecture supported") { isArchitectureSupported(architecture) }
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
+      onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
     }
 
   if (base == DockerBase.IRON_BANK) {
@@ -348,8 +360,8 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) {
     expansions(architecture, base).findAll { it.key != 'build_date' }.each { k, v ->
       inputs.property(k, { v.toString() })
     }
-
-    onlyIf("$architecture supported") { isArchitectureSupported(architecture) }
+    Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
+    onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
   }
 
   // Register transformed context as a project artifact
@@ -386,6 +398,7 @@ private static List<String> generateTags(DockerBase base, Architecture architect
 }
 
 void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
+
   final TaskProvider<DockerBuildTask> buildDockerImageTask =
     tasks.register(taskName("build", architecture, base, "DockerImage"), DockerBuildTask) {
 
@@ -421,7 +434,9 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
         baseImages = [base.image]
       }
 
-      onlyIf("$architecture supported") { isArchitectureSupported(architecture) }
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
+      onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
+
     }
 
   if (base != DockerBase.IRON_BANK && base != DockerBase.CLOUD && base != DockerBase.CLOUD_ESS) {
@@ -467,8 +482,9 @@ void addBuildEssDockerImageTask(Architecture architecture) {
       baseImages = []
       tags = generateTags(base, architecture)
       platforms.add(architecture.dockerPlatform)
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
+      onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
 
-      onlyIf("$architecture supported") { isArchitectureSupported(architecture) }
     }
 
   tasks.named("assemble").configure {
@@ -489,11 +505,6 @@ for (final Architecture architecture : Architecture.values()) {
   addBuildEssDockerImageTask(architecture)
 }
 
-boolean isArchitectureSupported(Architecture architecture) {
-  Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
-  return serviceProvider.get().dockerAvailability.supportedArchitectures().contains(architecture)
-}
-
 def exportDockerImages = tasks.register("exportDockerImages")
 def exportCompressedDockerImages = tasks.register("exportCompressedDockerImages")
 
@@ -538,7 +549,8 @@ subprojects { Project subProject ->
         tarFile,
         "elasticsearch${base.suffix}:${architecture.classifier}"
       dependsOn(parent.path + ":" + buildTaskName)
-      onlyIf("$architecture supported") { isArchitectureSupported(architecture) }
+      Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME)
+      onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
     }
 
     exportDockerImages.configure {

From fbe2b1ab245afefcaef423139d906038a4cfb199 Mon Sep 17 00:00:00 2001
From: Andrei Verdes <andrei.verdes.devops@gmail.com>
Date: Wed, 5 Jun 2024 12:38:52 +0300
Subject: [PATCH 158/208] update package-security.asciidoc to include link to
 reset password (#109314)

---
 docs/reference/setup/install/package-security.asciidoc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/reference/setup/install/package-security.asciidoc b/docs/reference/setup/install/package-security.asciidoc
index 40bd49d064b43..3b9f4fc1740ed 100644
--- a/docs/reference/setup/install/package-security.asciidoc
+++ b/docs/reference/setup/install/package-security.asciidoc
@@ -11,8 +11,9 @@ the `elastic` built-in superuser.
 and TLS is enabled and configured with these keys and certificates.
 
 The password and certificate and keys are output to your terminal.
+You can reset the password for the `elastic` user with the <<reset-password,`elasticsearch-reset-password`>> command.
 
-We recommend storing the `elastic` password as an environment variable in your shell. Example:
+We recommend storing the `elastic` password as an environment variable in your shell. For example:
 
 [source,sh]
 ----

From 32dfd1866e7ca04d3bd55901f97c613bf9d382b4 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 5 Jun 2024 05:40:34 -0400
Subject: [PATCH 159/208] ESQL: Improve some casting code (#109309)

If we're going to need it then it should be less ugly.
---
 .../xpack/esql/core/expression/FieldAttribute.java        | 8 ++++----
 .../xpack/esql/core/expression/MetadataAttribute.java     | 8 ++++----
 .../xpack/esql/core/expression/ReferenceAttribute.java    | 8 ++++----
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
index 35fe402035f69..a6e713007a97f 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java
@@ -78,9 +78,9 @@ public FieldAttribute(
     }
 
     @SuppressWarnings("unchecked")
-    public <S extends StreamInput & PlanStreamInput> FieldAttribute(StreamInput in) throws IOException {
+    public FieldAttribute(StreamInput in) throws IOException {
         /*
-         * The funny casting dance with `<S extends...>` and `(S) in` is required
+         * The funny casting dance with `(StreamInput & PlanStreamInput) in` is required
          * because we're in esql-core here and the real PlanStreamInput is in
          * esql-proper. And because NamedWriteableRegistry.Entry needs StreamInput,
          * not a PlanStreamInput. And we need PlanStreamInput to handle Source
@@ -88,14 +88,14 @@ public <S extends StreamInput & PlanStreamInput> FieldAttribute(StreamInput in)
          * of esql-core.
          */
         this(
-            Source.readFrom((S) in),
+            Source.readFrom((StreamInput & PlanStreamInput) in),
             in.readOptionalWriteable(FieldAttribute::new),
             in.readString(),
             DataType.readFrom(in),
             in.readNamedWriteable(EsField.class),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
-            NameId.readFrom((S) in),
+            NameId.readFrom((StreamInput & PlanStreamInput) in),
             in.readBoolean()
         );
     }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
index ed9ad996137e6..eac3586cf139d 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
@@ -66,9 +66,9 @@ public MetadataAttribute(Source source, String name, DataType dataType, boolean
     }
 
     @SuppressWarnings("unchecked")
-    public <S extends StreamInput & PlanStreamInput> MetadataAttribute(StreamInput in) throws IOException {
+    public MetadataAttribute(StreamInput in) throws IOException {
         /*
-         * The funny casting dance with `<S extends...>` and `(S) in` is required
+         * The funny casting dance with `(StreamInput & PlanStreamInput) in` is required
          * because we're in esql-core here and the real PlanStreamInput is in
          * esql-proper. And because NamedWriteableRegistry.Entry needs StreamInput,
          * not a PlanStreamInput. And we need PlanStreamInput to handle Source
@@ -76,12 +76,12 @@ public <S extends StreamInput & PlanStreamInput> MetadataAttribute(StreamInput i
          * of esql-core.
          */
         this(
-            Source.readFrom((S) in),
+            Source.readFrom((StreamInput & PlanStreamInput) in),
             in.readString(),
             DataType.readFrom(in),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
-            NameId.readFrom((S) in),
+            NameId.readFrom((StreamInput & PlanStreamInput) in),
             in.readBoolean(),
             in.readBoolean()
         );
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
index 8bac20e9347bc..d9a70787a56ed 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java
@@ -43,9 +43,9 @@ public ReferenceAttribute(
     }
 
     @SuppressWarnings("unchecked")
-    public <S extends StreamInput & PlanStreamInput> ReferenceAttribute(StreamInput in) throws IOException {
+    public ReferenceAttribute(StreamInput in) throws IOException {
         /*
-         * The funny casting dance with `<S extends...>` and `(S) in` is required
+         * The funny casting dance with `(StreamInput & PlanStreamInput) in` is required
          * because we're in esql-core here and the real PlanStreamInput is in
          * esql-proper. And because NamedWriteableRegistry.Entry needs StreamInput,
          * not a PlanStreamInput. And we need PlanStreamInput to handle Source
@@ -53,12 +53,12 @@ public <S extends StreamInput & PlanStreamInput> ReferenceAttribute(StreamInput
          * of esql-core.
          */
         this(
-            Source.readFrom((S) in),
+            Source.readFrom((StreamInput & PlanStreamInput) in),
             in.readString(),
             DataType.readFrom(in),
             in.readOptionalString(),
             in.readEnum(Nullability.class),
-            NameId.readFrom((S) in),
+            NameId.readFrom((StreamInput & PlanStreamInput) in),
             in.readBoolean()
         );
     }

From 564549af8fe73e270f1341bbfc14b40cc9da4a1c Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Wed, 5 Jun 2024 11:43:07 +0100
Subject: [PATCH 160/208] Expose the bitset filter cache in the
 MappingParserContext (#109298)

Add the bitset filter cache in the MappingParserContext
---
 .../index/mapper/MapperServiceFactory.java    | 11 ++++++++++
 .../search/QueryParserHelperBenchmark.java    |  1 +
 .../metadata/IndexMetadataVerifier.java       |  3 +++
 .../org/elasticsearch/index/IndexModule.java  |  3 +++
 .../org/elasticsearch/index/IndexService.java |  3 ++-
 .../index/mapper/MapperService.java           |  8 +++++++-
 .../index/mapper/MappingParserContext.java    | 17 +++++++++++++---
 .../elasticsearch/index/codec/CodecTests.java | 11 ++++++++++
 .../index/mapper/MappingParserTests.java      | 13 +++++++++++-
 .../index/mapper/ParametrizedMapperTests.java |  5 ++++-
 .../index/mapper/TypeParsersTests.java        | 10 ++++++++--
 .../query/SearchExecutionContextTests.java    |  5 ++++-
 .../elasticsearch/index/MapperTestUtils.java  | 11 ++++++++++
 .../index/engine/TranslogHandler.java         |  3 +++
 .../index/mapper/MapperServiceTestCase.java   |  9 +++++++++
 .../mapper/TestDocumentParserContext.java     |  5 ++++-
 .../aggregations/AggregatorTestCase.java      |  5 ++++-
 .../test/AbstractBuilderTestCase.java         | 20 ++++++++-----------
 18 files changed, 119 insertions(+), 24 deletions(-)

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java
index 70e9fe424e77b..68b31481e17f3 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.benchmark.index.mapper;
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.util.Accountable;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -21,10 +22,12 @@
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.LowercaseNormalizer;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.script.Script;
@@ -52,6 +55,13 @@ public static MapperService create(String mappings) {
         MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
 
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {}
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
+        });
         MapperService mapperService = new MapperService(
             () -> TransportVersion.current(),
             indexSettings,
@@ -73,6 +83,7 @@ public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
             },
+            bitsetFilterCache::getBitSetProducer,
             MapperMetrics.NOOP
         );
 
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java
index 14f6fe6501a73..cff15d9c36d34 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java
@@ -189,6 +189,7 @@ public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
             },
+            query -> { throw new UnsupportedOperationException(); },
             MapperMetrics.NOOP
         );
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
index 0124f23a1156d..e774d7e4d552d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
@@ -187,6 +187,9 @@ protected TokenStreamComponents createComponents(String fieldName) {
                     () -> null,
                     indexSettings.getMode().idFieldMapperWithoutFieldData(),
                     scriptService,
+                    query -> {
+                        throw new UnsupportedOperationException("IndexMetadataVerifier");
+                    },
                     mapperMetrics
                 )
             ) {
diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java
index ff8db4bacef8c..fa2a9f0f35259 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -652,6 +652,9 @@ public MapperService newIndexMapperService(
             },
             indexSettings.getMode().idFieldMapperWithoutFieldData(),
             scriptService,
+            query -> {
+                throw new UnsupportedOperationException("no index query shard context available");
+            },
             mapperMetrics
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java
index 1712f824a132c..0605e36b2ea4b 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexService.java
@@ -212,6 +212,7 @@ public IndexService(
         this.indexAnalyzers = indexAnalyzers;
         if (needsMapperService(indexSettings, indexCreationContext)) {
             assert indexAnalyzers != null;
+            this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
             this.mapperService = new MapperService(
                 clusterService,
                 indexSettings,
@@ -223,6 +224,7 @@ public IndexService(
                 () -> newSearchExecutionContext(0, 0, null, System::currentTimeMillis, null, emptyMap()),
                 idFieldMapper,
                 scriptService,
+                bitsetFilterCache::getBitSetProducer,
                 mapperMetrics
             );
             this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService);
@@ -238,7 +240,6 @@ public IndexService(
                 this.indexSortSupplier = () -> null;
             }
             indexFieldData.setListener(new FieldDataCacheListener(this));
-            this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
             this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool));
             this.indexCache = new IndexCache(queryCache, bitsetFilterCache);
         } else {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index d3665c3b978bd..e5dc95ddbc2a0 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
@@ -167,6 +169,7 @@ public MapperService(
         Supplier<SearchExecutionContext> searchExecutionContextSupplier,
         IdFieldMapper idFieldMapper,
         ScriptCompiler scriptCompiler,
+        Function<Query, BitSetProducer> bitSetProducer,
         MapperMetrics mapperMetrics
     ) {
         this(
@@ -179,6 +182,7 @@ public MapperService(
             searchExecutionContextSupplier,
             idFieldMapper,
             scriptCompiler,
+            bitSetProducer,
             mapperMetrics
         );
     }
@@ -194,6 +198,7 @@ public MapperService(
         Supplier<SearchExecutionContext> searchExecutionContextSupplier,
         IdFieldMapper idFieldMapper,
         ScriptCompiler scriptCompiler,
+        Function<Query, BitSetProducer> bitSetProducer,
         MapperMetrics mapperMetrics
     ) {
         super(indexSettings);
@@ -210,7 +215,8 @@ public MapperService(
             scriptCompiler,
             indexAnalyzers,
             indexSettings,
-            idFieldMapper
+            idFieldMapper,
+            bitSetProducer
         );
         this.documentParser = new DocumentParser(parserConfiguration, this.mappingParserContextSupplier.get());
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java
index 88df87859ccc2..3f614d4346fd4 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
@@ -37,6 +39,7 @@ public class MappingParserContext {
     private final IndexAnalyzers indexAnalyzers;
     private final IndexSettings indexSettings;
     private final IdFieldMapper idFieldMapper;
+    private final Function<Query, BitSetProducer> bitSetProducer;
     private final long mappingObjectDepthLimit;
     private long mappingObjectDepth = 0;
 
@@ -50,7 +53,8 @@ public MappingParserContext(
         ScriptCompiler scriptCompiler,
         IndexAnalyzers indexAnalyzers,
         IndexSettings indexSettings,
-        IdFieldMapper idFieldMapper
+        IdFieldMapper idFieldMapper,
+        Function<Query, BitSetProducer> bitSetProducer
     ) {
         this.similarityLookupService = similarityLookupService;
         this.typeParsers = typeParsers;
@@ -63,6 +67,7 @@ public MappingParserContext(
         this.indexSettings = indexSettings;
         this.idFieldMapper = idFieldMapper;
         this.mappingObjectDepthLimit = indexSettings.getMappingDepthLimit();
+        this.bitSetProducer = bitSetProducer;
     }
 
     public IndexAnalyzers getIndexAnalyzers() {
@@ -132,6 +137,10 @@ public ScriptCompiler scriptCompiler() {
         return scriptCompiler;
     }
 
+    public BitSetProducer bitSetProducer(Query query) {
+        return bitSetProducer.apply(query);
+    }
+
     void incrementMappingObjectDepth() throws MapperParsingException {
         mappingObjectDepth++;
         if (mappingObjectDepth > mappingObjectDepthLimit) {
@@ -159,7 +168,8 @@ private static class MultiFieldParserContext extends MappingParserContext {
                 in.scriptCompiler,
                 in.indexAnalyzers,
                 in.indexSettings,
-                in.idFieldMapper
+                in.idFieldMapper,
+                in.bitSetProducer
             );
         }
 
@@ -188,7 +198,8 @@ private static class DynamicTemplateParserContext extends MappingParserContext {
                 in.scriptCompiler,
                 in.indexAnalyzers,
                 in.indexSettings,
-                in.idFieldMapper
+                in.idFieldMapper,
+                in.bitSetProducer
             );
             this.dateFormatter = dateFormatter;
         }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
index ffb3cc1943bff..3c687f1792d0d 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
@@ -19,15 +19,18 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
+import org.apache.lucene.util.Accountable;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.script.ScriptCompiler;
@@ -107,6 +110,13 @@ private CodecService createCodecService() throws IOException {
             Collections.emptyMap(),
             MapperPlugin.NOOP_FIELD_FILTER
         );
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {}
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
+        });
         MapperService service = new MapperService(
             () -> TransportVersion.current(),
             settings,
@@ -117,6 +127,7 @@ private CodecService createCodecService() throws IOException {
             () -> null,
             settings.getMode().idFieldMapperWithoutFieldData(),
             ScriptCompiler.NONE,
+            bitsetFilterCache::getBitSetProducer,
             MapperMetrics.NOOP
         );
         return new CodecService(service, BigArrays.NON_RECYCLING_INSTANCE);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
index abe8e820acae8..aa22a345c5cec 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.util.Accountable;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -17,6 +18,8 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.script.ScriptService;
@@ -43,6 +46,13 @@ private static MappingParser createMappingParser(Settings settings, IndexVersion
         IndexAnalyzers indexAnalyzers = createIndexAnalyzers();
         SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, Collections.emptyMap());
         MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {}
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
+        });
         Supplier<MappingParserContext> mappingParserContextSupplier = () -> new MappingParserContext(
             similarityService::getSimilarity,
             type -> mapperRegistry.getMapperParser(type, indexSettings.getIndexVersionCreated()),
@@ -55,7 +65,8 @@ private static MappingParser createMappingParser(Settings settings, IndexVersion
             scriptService,
             indexAnalyzers,
             indexSettings,
-            indexSettings.getMode().idFieldMapperWithoutFieldData()
+            indexSettings.getMode().idFieldMapperWithoutFieldData(),
+            bitsetFilterCache::getBitSetProducer
         );
 
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java
index b1b7f80ba865f..0ec1997ae652e 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java
@@ -277,7 +277,10 @@ private static TestMapper fromMapping(
             ScriptCompiler.NONE,
             mapperService.getIndexAnalyzers(),
             mapperService.getIndexSettings(),
-            mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData()
+            mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData(),
+            query -> {
+                throw new UnsupportedOperationException();
+            }
         );
         if (fromDynamicTemplate) {
             pc = pc.createDynamicTemplateContext(null);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
index 2b704a25e2232..035466d93ab06 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
@@ -97,7 +97,10 @@ public void testMultiFieldWithinMultiField() throws IOException {
             ScriptCompiler.NONE,
             mapperService.getIndexAnalyzers(),
             mapperService.getIndexSettings(),
-            ProvidedIdFieldMapper.NO_FIELD_DATA
+            ProvidedIdFieldMapper.NO_FIELD_DATA,
+            query -> {
+                throw new UnsupportedOperationException();
+            }
         );
 
         TextFieldMapper.PARSER.parse("some-field", fieldNode, olderContext);
@@ -128,7 +131,10 @@ public void testMultiFieldWithinMultiField() throws IOException {
             ScriptCompiler.NONE,
             mapperService.getIndexAnalyzers(),
             mapperService.getIndexSettings(),
-            ProvidedIdFieldMapper.NO_FIELD_DATA
+            ProvidedIdFieldMapper.NO_FIELD_DATA,
+            query -> {
+                throw new UnsupportedOperationException();
+            }
         );
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
index 6d8a22e7850e4..9cd1df700a618 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
@@ -548,7 +548,10 @@ private static MapperService createMapperService(IndexSettings indexSettings, Ma
                 ScriptCompiler.NONE,
                 indexAnalyzers,
                 indexSettings,
-                indexSettings.getMode().buildIdFieldMapper(() -> true)
+                indexSettings.getMode().buildIdFieldMapper(() -> true),
+                query -> {
+                    throw new UnsupportedOperationException();
+                }
             )
         );
         when(mapperService.isMultiField(anyString())).then(
diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
index 5025299b09b64..913caba615a67 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
@@ -8,15 +8,18 @@
 
 package org.elasticsearch.index;
 
+import org.apache.lucene.util.Accountable;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.script.ScriptCompiler;
@@ -58,6 +61,13 @@ public static MapperService newMapperService(
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings);
         IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, finalSettings).indexAnalyzers;
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap());
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {}
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
+        });
         return new MapperService(
             () -> TransportVersion.current(),
             indexSettings,
@@ -68,6 +78,7 @@ public static MapperService newMapperService(
             () -> null,
             indexSettings.getMode().idFieldMapperWithoutFieldData(),
             ScriptCompiler.NONE,
+            bitsetFilterCache::getBitSetProducer,
             MapperMetrics.NOOP
         );
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
index c2da7a561c041..dc626a3228685 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
@@ -55,6 +55,9 @@ public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings ind
             () -> null,
             indexSettings.getMode().idFieldMapperWithoutFieldData(),
             null,
+            query -> {
+                throw new UnsupportedOperationException("The bitset filter cache is not available in translog operations");
+            },
             MapperMetrics.NOOP
         );
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
index 7d13e33be0db7..388d8d6fa6ffd 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
@@ -254,6 +254,14 @@ public MapperService build() {
                 getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList())
             ).getMapperRegistry();
 
+            BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
+                @Override
+                public void onCache(ShardId shardId, Accountable accountable) {}
+
+                @Override
+                public void onRemoval(ShardId shardId, Accountable accountable) {}
+            });
+
             return new MapperService(
                 () -> TransportVersion.current(),
                 indexSettings,
@@ -266,6 +274,7 @@ public MapperService build() {
                 },
                 indexSettings.getMode().buildIdFieldMapper(idFieldDataEnabled),
                 scriptCompiler,
+                bitsetFilterCache::getBitSetProducer,
                 mapperMetrics
             );
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java
index d4c238322e28a..5243ef85cdb76 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java
@@ -63,7 +63,10 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou
                 null,
                 (type, name) -> Lucene.STANDARD_ANALYZER,
                 MapperTestCase.createIndexSettings(IndexVersion.current(), settings),
-                null
+                null,
+                query -> {
+                    throw new UnsupportedOperationException();
+                }
             ),
             source,
             mappingLookup.getMapping().getRoot(),
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index 134352a4f8af4..d39a8df80c26d 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -1284,7 +1284,10 @@ private static class MockParserContext extends MappingParserContext {
                 ScriptCompiler.NONE,
                 null,
                 indexSettings,
-                null
+                null,
+                query -> {
+                    throw new UnsupportedOperationException();
+                }
             );
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
index 6ef8d3d8a6a1b..271df2a971fb1 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
@@ -467,6 +467,13 @@ private static class ServiceHolder implements Closeable {
             IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings);
             scriptService = new MockScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts);
             similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap());
+            this.bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
+                @Override
+                public void onCache(ShardId shardId, Accountable accountable) {}
+
+                @Override
+                public void onRemoval(ShardId shardId, Accountable accountable) {}
+            });
             MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
             mapperService = new MapperService(
                 clusterService,
@@ -478,23 +485,12 @@ private static class ServiceHolder implements Closeable {
                 () -> createShardContext(null),
                 idxSettings.getMode().idFieldMapperWithoutFieldData(),
                 ScriptCompiler.NONE,
+                bitsetFilterCache::getBitSetProducer,
                 MapperMetrics.NOOP
             );
             IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
             });
             indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache, new NoneCircuitBreakerService());
-            bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
-                @Override
-                public void onCache(ShardId shardId, Accountable accountable) {
-
-                }
-
-                @Override
-                public void onRemoval(ShardId shardId, Accountable accountable) {
-
-                }
-            });
-
             if (registerType) {
                 mapperService.merge(
                     "_doc",

From b8b6513fafb38fb34d5a6f5d1a955e6c811949bb Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Wed, 5 Jun 2024 12:48:30 +0200
Subject: [PATCH 161/208] Stop allocating BytesRefBuilder for each StreamOutput
 (#109251)

We can use a thread-local here, no need to allocate a fresh instance
for each output when they mostly go unused anyway.
---
 .../java/org/elasticsearch/common/io/stream/StreamOutput.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 33fb000c1bca2..833e7f27852c8 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -370,11 +370,12 @@ public void writeOptionalText(@Nullable Text text) throws IOException {
         }
     }
 
-    private final BytesRefBuilder spare = new BytesRefBuilder();
+    private static final ThreadLocal<BytesRefBuilder> spareBytesRefBuilder = ThreadLocal.withInitial(BytesRefBuilder::new);
 
     public void writeText(Text text) throws IOException {
         if (text.hasBytes() == false) {
             final String string = text.string();
+            var spare = spareBytesRefBuilder.get();
             spare.copyChars(string);
             writeInt(spare.length());
             write(spare.bytes(), 0, spare.length());

From db130982d94ca9f5952c6c2a5436be048ccfbf38 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 5 Jun 2024 07:02:37 -0400
Subject: [PATCH 162/208] ESQL: Link to the function guide (#109351)

This adds some links to the function guide in the javadoc for the
superclass of scalar functions.
---
 .../expression/function/scalar/EsqlScalarFunction.java | 10 ++++++++++
 .../scalar/convert/AbstractConvertFunction.java        |  4 ++++
 .../scalar/multivalue/AbstractMultivalueFunction.java  |  4 ++++
 3 files changed, 18 insertions(+)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java
index 797a4c31f0f6c..4f991af54ecff 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java
@@ -14,6 +14,16 @@
 
 import java.util.List;
 
+/**
+ * A {@code ScalarFunction} is a {@code Function} that takes values from some
+ * operation and converts each to another value. An example would be
+ * {@code ABS()}, which takes one value at a time, applies a function to the
+ * value (abs) and returns a new value.
+ * <p>
+ *     We have a guide for writing these in the javadoc for
+ *     {@link org.elasticsearch.xpack.esql.expression.function.scalar}.
+ * </p>
+ */
 public abstract class EsqlScalarFunction extends ScalarFunction implements EvaluatorMapper {
 
     protected EsqlScalarFunction(Source source) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
index 955ce1646813f..f1d0256a1f1c7 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
@@ -38,6 +38,10 @@
 
 /**
  * Base class for functions that converts a field into a function-specific type.
+ * <p>
+ *     We have a guide for writing these in the javadoc for
+ *     {@link org.elasticsearch.xpack.esql.expression.function.scalar}.
+ * </p>
  */
 public abstract class AbstractConvertFunction extends UnaryScalarFunction {
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java
index 2ceedd14d6fd8..5aa6dad7b2a5b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java
@@ -19,6 +19,10 @@
 
 /**
  * Base class for functions that reduce multivalued fields into single valued fields.
+ * <p>
+ *     We have a guide for writing these in the javadoc for
+ *     {@link org.elasticsearch.xpack.esql.expression.function.scalar}.
+ * </p>
  */
 public abstract class AbstractMultivalueFunction extends UnaryScalarFunction {
     protected AbstractMultivalueFunction(Source source, Expression field) {

From a36fef011c6167b83d09293d3e9672598f567a01 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 5 Jun 2024 07:03:22 -0400
Subject: [PATCH 163/208] ESQL: Entirely drop `version` field (#109376)

This field is no longer supported.
---
 .../xpack/esql/action/RequestXContent.java    |  2 --
 .../esql/action/EsqlQueryRequestTests.java    | 33 +------------------
 .../rest-api-spec/test/esql/10_basic.yml      | 12 +++++++
 3 files changed, 13 insertions(+), 34 deletions(-)

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java
index 9ffd48d9d0c3b..793f453d5ebf5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java
@@ -46,7 +46,6 @@ final class RequestXContent {
         PARAM_PARSER.declareString(constructorArg(), TYPE);
     }
 
-    static final ParseField ESQL_VERSION_FIELD = new ParseField("version");
     static final ParseField QUERY_FIELD = new ParseField("query");
     private static final ParseField COLUMNAR_FIELD = new ParseField("columnar");
     private static final ParseField FILTER_FIELD = new ParseField("filter");
@@ -74,7 +73,6 @@ static EsqlQueryRequest parseAsync(XContentParser parser) {
     }
 
     private static void objectParserCommon(ObjectParser<EsqlQueryRequest, ?> parser) {
-        parser.declareString((str, consumer) -> {}, ESQL_VERSION_FIELD);
         parser.declareString(EsqlQueryRequest::query, QUERY_FIELD);
         parser.declareBoolean(EsqlQueryRequest::columnar, COLUMNAR_FIELD);
         parser.declareObject(EsqlQueryRequest::filter, (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p), FILTER_FIELD);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
index 5fafafe34bd23..6328853eea3c6 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
@@ -160,41 +160,10 @@ public void testRejectUnknownFields() {
             }""", "unknown field [asdf]");
     }
 
-    public void testAnyVersionIsValid() throws IOException {
-        String validVersionString = randomAlphaOfLength(5);
-
-        String json = String.format(Locale.ROOT, """
-            {
-                "version": "%s",
-                "query": "ROW x = 1"
-            }
-            """, validVersionString);
-
-        EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
-        assertNull(request.validate());
-
-        request = parseEsqlQueryRequestAsync(json);
-        assertNull(request.validate());
-    }
-
-    public void testMissingVersionIsValid() throws IOException {
-        String missingVersion = randomBoolean() ? "" : ", \"version\": \"\"";
-        String json = String.format(Locale.ROOT, """
-            {
-                "columnar": true,
-                "query": "row x = 1"
-                %s
-            }""", missingVersion);
-
-        EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
-        assertNull(request.validate());
-    }
-
     public void testMissingQueryIsNotValid() throws IOException {
         String json = """
             {
-                "columnar": true,
-                "version": "snapshot"
+                "columnar": true
             }""";
         EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
         assertNotNull(request.validate());
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml
index 52d390e7b288b..ab0261d916630 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml
@@ -333,3 +333,15 @@ setup:
   - match: {values.0: ["1",2.0,null,true,123,123]}
   - match: {values.1: ["1",2.0,null,true,123,123]}
   - match: {values.2: ["1",2.0,null,true,123,123]}
+
+---
+version is not allowed:
+  - requires:
+      cluster_features: ["gte_v8.14.0"]
+      reason: version allowed in 8.13.latest
+  - do:
+      catch: /unknown field \[version\]/
+      esql.query:
+        body:
+          query: 'from test'
+          version: cat

From 3ff27a999271f758d81d1db4aa21ec077a4634d1 Mon Sep 17 00:00:00 2001
From: Luigi Dell'Aquila <luigi.dellaquila@gmail.com>
Date: Wed, 5 Jun 2024 13:15:25 +0200
Subject: [PATCH 164/208] ES|QL: Fix MvAppend tests (#109383)

---
 muted-tests.yml                                           | 3 ---
 .../function/scalar/multivalue/MvAppendTests.java         | 8 ++++----
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index be4845ccec9e9..249541ff97926 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -53,9 +53,6 @@ tests:
 - class: "org.elasticsearch.client.RestClientSingleHostIntegTests"
   issue: "https://github.com/elastic/elasticsearch/issues/102717"
   method: "testRequestResetAndAbort"
-- class: org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppendTests
-  issue: https://github.com/elastic/elasticsearch/issues/109373
-  method: testEvaluateBlockWithoutNulls
 - class: "org.elasticsearch.xpack.deprecation.DeprecationHttpIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108628"
   method: "testDeprecatedSettingsReturnWarnings"
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
index 07dab82c50607..6361360652a87 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java
@@ -235,8 +235,8 @@ private static void bytesRefs(List<TestCaseSupplier> suppliers) {
         }));
 
         suppliers.add(new TestCaseSupplier(List.of(DataType.GEO_SHAPE, DataType.GEO_SHAPE), () -> {
-            List<Object> field1 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))));
-            List<Object> field2 = randomList(1, 10, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))));
+            List<Object> field1 = randomList(1, 5, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))));
+            List<Object> field2 = randomList(1, 5, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean()))));
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(
@@ -251,8 +251,8 @@ private static void bytesRefs(List<TestCaseSupplier> suppliers) {
         }));
 
         suppliers.add(new TestCaseSupplier(List.of(DataType.CARTESIAN_SHAPE, DataType.CARTESIAN_SHAPE), () -> {
-            List<Object> field1 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))));
-            List<Object> field2 = randomList(1, 10, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))));
+            List<Object> field1 = randomList(1, 5, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))));
+            List<Object> field2 = randomList(1, 5, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean()))));
             var result = new ArrayList<>(field1);
             result.addAll(field2);
             return new TestCaseSupplier.TestCase(

From cd84749d8726b3b2d80d4731be9ba33d002161b2 Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Wed, 5 Jun 2024 07:38:58 -0400
Subject: [PATCH 165/208] AwaitsFix:
 https://github.com/elastic/elasticsearch/issues/109391

---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index 249541ff97926..3d64f87144bd3 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -56,6 +56,9 @@ tests:
 - class: "org.elasticsearch.xpack.deprecation.DeprecationHttpIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108628"
   method: "testDeprecatedSettingsReturnWarnings"
+- class: "org.elasticsearch.xpack.inference.InferenceCrudIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/109391"
+  method: "testDeleteEndpointWhileReferencedByPipeline"
 
 # Examples:
 #

From fdb5058b13abbbc5ce3a5d7c0ab1a7878c72176c Mon Sep 17 00:00:00 2001
From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com>
Date: Wed, 5 Jun 2024 08:25:25 -0400
Subject: [PATCH 166/208] [ML] Inference API rate limit queuing logic refactor
 (#107706)

* Adding new executor

* Adding in queuing logic

* working tests

* Added cleanup task

* Update docs/changelog/107706.yaml

* Updating yml

* deregistering callbacks for settings changes

* Cleaning up code

* Update docs/changelog/107706.yaml

* Fixing rate limit settings bug and only sleeping least amount

* Removing debug logging

* Removing commented code

* Renaming feedback

* fixing tests

* Updating docs and validation

* Fixing source blocks

* Adjusting cancel logic

* Reformatting ascii

* Addressing feedback

* adding rate limiting for google embeddings and mistral

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 docs/changelog/107706.yaml                    |   5 +
 .../inference/put-inference.asciidoc          | 289 ++++++----
 .../org/elasticsearch/core/TimeValue.java     |   7 +
 .../common/unit/TimeValueTests.java           |   7 +
 .../action/cohere/CohereActionCreator.java    |   1 +
 .../action/cohere/CohereEmbeddingsAction.java |   1 +
 ...eAiStudioChatCompletionRequestManager.java |   6 +-
 ...AzureAiStudioEmbeddingsRequestManager.java |   6 +-
 .../AzureOpenAiCompletionRequestManager.java  |   6 +-
 .../AzureOpenAiEmbeddingsRequestManager.java  |   6 +-
 .../http/sender/BaseRequestManager.java       |   7 +-
 .../CohereCompletionRequestManager.java       |   6 +-
 .../CohereEmbeddingsRequestManager.java       |   6 +-
 .../sender/CohereRerankRequestManager.java    |   6 +-
 .../sender/ExecutableInferenceRequest.java    |   3 +-
 ...oogleAiStudioCompletionRequestManager.java |   6 +-
 ...oogleAiStudioEmbeddingsRequestManager.java |   6 +-
 .../http/sender/HttpRequestSender.java        |  20 +-
 .../sender/HuggingFaceRequestManager.java     |  14 +-
 .../http/sender/InferenceRequest.java         |   4 +-
 .../MistralEmbeddingsRequestManager.java      |   6 +-
 .../external/http/sender/NoopTask.java        |  52 --
 .../OpenAiCompletionRequestManager.java       |   6 +-
 .../OpenAiEmbeddingsRequestManager.java       |   6 +-
 .../http/sender/RequestExecutorService.java   | 500 ++++++++++++------
 .../RequestExecutorServiceSettings.java       |  86 ++-
 .../external/http/sender/RequestManager.java  |   8 +-
 .../external/http/sender/RequestTask.java     |   2 +-
 .../http/sender/SingleRequestManager.java     |  48 --
 .../inference/services/SenderService.java     |   2 +-
 .../azureaistudio/AzureAiStudioService.java   |   2 +-
 .../AzureAiStudioServiceSettings.java         |  10 +-
 .../azureopenai/AzureOpenAiService.java       |  10 +-
 .../AzureOpenAiCompletionModel.java           |   6 +-
 .../AzureOpenAiCompletionServiceSettings.java |  19 +-
 .../AzureOpenAiEmbeddingsServiceSettings.java |  12 +-
 .../services/cohere/CohereService.java        |  15 +-
 .../cohere/CohereServiceSettings.java         |  14 +-
 .../completion/CohereCompletionModel.java     |   6 +-
 .../CohereCompletionServiceSettings.java      |  14 +-
 .../googleaistudio/GoogleAiStudioService.java |   6 +-
 .../GoogleAiStudioCompletionModel.java        |   6 +-
 ...ogleAiStudioCompletionServiceSettings.java |  14 +-
 .../GoogleAiStudioEmbeddingsModel.java        |   6 +-
 ...ogleAiStudioEmbeddingsServiceSettings.java |  14 +-
 .../huggingface/HuggingFaceBaseService.java   |  19 +-
 .../huggingface/HuggingFaceService.java       |  15 +-
 .../HuggingFaceServiceSettings.java           |  13 +-
 .../elser/HuggingFaceElserModel.java          |   6 +-
 .../elser/HuggingFaceElserService.java        |   6 +-
 .../HuggingFaceElserServiceSettings.java      |  14 +-
 .../HuggingFaceEmbeddingsModel.java           |   6 +-
 .../MistralEmbeddingsServiceSettings.java     |  11 +-
 .../services/openai/OpenAiService.java        |   3 +-
 .../completion/OpenAiChatCompletionModel.java |   6 +-
 .../OpenAiChatCompletionServiceSettings.java  |  14 +-
 .../OpenAiEmbeddingsServiceSettings.java      |  21 +-
 .../services/settings/RateLimitSettings.java  |  15 +-
 .../AzureAiStudioActionAndCreatorTests.java   |   5 +-
 .../AzureOpenAiActionCreatorTests.java        |  19 +-
 .../AzureOpenAiCompletionActionTests.java     |   3 +-
 .../AzureOpenAiEmbeddingsActionTests.java     |   3 +-
 .../cohere/CohereActionCreatorTests.java      |   7 +-
 .../cohere/CohereCompletionActionTests.java   |   6 +-
 .../cohere/CohereEmbeddingsActionTests.java   |   4 +-
 .../GoogleAiStudioCompletionActionTests.java  |   4 +-
 .../GoogleAiStudioEmbeddingsActionTests.java  |   2 +-
 .../HuggingFaceActionCreatorTests.java        |  13 +-
 .../openai/OpenAiActionCreatorTests.java      |  23 +-
 .../OpenAiChatCompletionActionTests.java      |   5 +-
 .../openai/OpenAiEmbeddingsActionTests.java   |   2 +-
 .../http/sender/BaseRequestManagerTests.java  | 122 +++++
 .../http/sender/HttpRequestSenderTests.java   |  29 +-
 .../RequestExecutorServiceSettingsTests.java  |  12 +
 .../sender/RequestExecutorServiceTests.java   | 345 ++++++++----
 ...torTests.java => RequestManagerTests.java} |  38 +-
 .../sender/SingleRequestManagerTests.java     |  27 -
 .../services/SenderServiceTests.java          |   9 +-
 .../AzureAiStudioServiceTests.java            |   5 +-
 ...dioChatCompletionServiceSettingsTests.java |   3 +-
 ...iStudioEmbeddingsServiceSettingsTests.java |   2 +-
 .../azureopenai/AzureOpenAiServiceTests.java  |   5 +-
 ...eOpenAiCompletionServiceSettingsTests.java |  16 +-
 ...eOpenAiEmbeddingsServiceSettingsTests.java |   4 +-
 .../services/cohere/CohereServiceTests.java   |   5 +-
 .../CohereCompletionModelTests.java           |   4 +-
 .../CohereCompletionServiceSettingsTests.java |  19 +-
 .../CohereEmbeddingsServiceSettingsTests.java |  15 -
 .../CohereRerankServiceSettingsTests.java     |  14 -
 .../GoogleAiStudioServiceTests.java           |   5 +-
 .../GoogleAiStudioCompletionModelTests.java   |   4 +-
 ...iStudioCompletionServiceSettingsTests.java |  18 +-
 ...iStudioEmbeddingsServiceSettingsTests.java |  21 +-
 .../HuggingFaceBaseServiceTests.java          |   9 +-
 .../HuggingFaceServiceSettingsTests.java      |  38 +-
 .../HuggingFaceElserServiceSettingsTests.java |  33 +-
 .../services/mistral/MistralServiceTests.java |   5 +-
 ...MistralEmbeddingsServiceSettingsTests.java |  12 -
 .../services/openai/OpenAiServiceTests.java   |   5 +-
 ...nAiChatCompletionServiceSettingsTests.java |  37 +-
 .../OpenAiEmbeddingsServiceSettingsTests.java |   4 +-
 .../settings/RateLimitSettingsTests.java      |  20 +-
 102 files changed, 1487 insertions(+), 925 deletions(-)
 create mode 100644 docs/changelog/107706.yaml
 delete mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java
 delete mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManager.java
 create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManagerTests.java
 rename x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/{ExecutableRequestCreatorTests.java => RequestManagerTests.java} (56%)
 delete mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManagerTests.java

diff --git a/docs/changelog/107706.yaml b/docs/changelog/107706.yaml
new file mode 100644
index 0000000000000..76b7f662bf0e0
--- /dev/null
+++ b/docs/changelog/107706.yaml
@@ -0,0 +1,5 @@
+pr: 107706
+summary: Add rate limiting support for the Inference API
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc
index 354cee3f6a990..f805bc0cc92f7 100644
--- a/docs/reference/inference/put-inference.asciidoc
+++ b/docs/reference/inference/put-inference.asciidoc
@@ -7,21 +7,17 @@ experimental[]
 Creates an {infer} endpoint to perform an {infer} task.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure
-OpenAI, Google AI Studio or Hugging Face. For built-in models and models
-uploaded though Eland, the {infer} APIs offer an alternative way to use and
-manage trained models. However, if you do not plan to use the {infer} APIs to
-use these models or if you want to use non-NLP models, use the
+{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure OpenAI, Google AI Studio or Hugging Face.
+For built-in models and models uploaded though Eland, the {infer} APIs offer an alternative way to use and manage trained models.
+However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the
 <<ml-df-trained-models-apis>>.
 
-
 [discrete]
 [[put-inference-api-request]]
 ==== {api-request-title}
 
 `PUT /_inference/<task_type>/<inference_id>`
 
-
 [discrete]
 [[put-inference-api-prereqs]]
 ==== {api-prereq-title}
@@ -29,7 +25,6 @@ use these models or if you want to use non-NLP models, use the
 * Requires the `manage_inference` <<privileges-list-cluster,cluster privilege>>
 (the built-in `inference_admin` role grants this privilege)
 
-
 [discrete]
 [[put-inference-api-desc]]
 ==== {api-description-title}
@@ -48,25 +43,23 @@ The following services are available through the {infer} API:
 * Hugging Face
 * OpenAI
 
-
 [discrete]
 [[put-inference-api-path-params]]
 ==== {api-path-parms-title}
 
-
 `<inference_id>`::
 (Required, string)
 The unique identifier of the {infer} endpoint.
 
 `<task_type>`::
 (Required, string)
-The type of the {infer} task that the model will perform. Available task types:
+The type of the {infer} task that the model will perform.
+Available task types:
 * `completion`,
 * `rerank`,
 * `sparse_embedding`,
 * `text_embedding`.
 
-
 [discrete]
 [[put-inference-api-request-body]]
 ==== {api-request-body-title}
@@ -78,21 +71,18 @@ Available services:
 
 * `azureopenai`: specify the `completion` or `text_embedding` task type to use the Azure OpenAI service.
 * `azureaistudio`: specify the `completion` or `text_embedding` task type to use the Azure AI Studio service.
-* `cohere`: specify the `completion`, `text_embedding` or the `rerank` task type to use the
-Cohere service.
-* `elasticsearch`: specify the `text_embedding` task type to use the E5
-built-in model or text embedding models uploaded by Eland.
+* `cohere`: specify the `completion`, `text_embedding` or the `rerank` task type to use the Cohere service.
+* `elasticsearch`: specify the `text_embedding` task type to use the E5 built-in model or text embedding models uploaded by Eland.
 * `elser`: specify the `sparse_embedding` task type to use the ELSER service.
 * `googleaistudio`: specify the `completion` task to use the Google AI Studio service.
-* `hugging_face`: specify the `text_embedding` task type to use the Hugging Face
-service.
-* `openai`: specify the `completion` or `text_embedding` task type to use the
-OpenAI service.
+* `hugging_face`: specify the `text_embedding` task type to use the Hugging Face service.
+* `openai`: specify the `completion` or `text_embedding` task type to use the OpenAI service.
 
 
 `service_settings`::
 (Required, object)
-Settings used to install the {infer} model. These settings are specific to the
+Settings used to install the {infer} model.
+These settings are specific to the
 `service` you specified.
 +
 .`service_settings` for the `azureaistudio` service
@@ -104,11 +94,10 @@ Settings used to install the {infer} model. These settings are specific to the
 A valid API key of your Azure AI Studio model deployment.
 This key can be found on the overview page for your deployment in the management section of your https://ai.azure.com/[Azure AI Studio] account.
 
-IMPORTANT: You need to provide the API key only once, during the {infer} model
-creation. The <<get-inference-api>> does not retrieve your API key. After
-creating the {infer} model, you cannot change the associated API key. If you
-want to use a different API key, delete the {infer} model and recreate it with
-the same name and the updated API key.
+IMPORTANT: You need to provide the API key only once, during the {infer} model creation.
+The <<get-inference-api>> does not retrieve your API key.
+After creating the {infer} model, you cannot change the associated API key.
+If you want to use a different API key, delete the {infer} model and recreate it with the same name and the updated API key.
 
 `target`:::
 (Required, string)
@@ -142,11 +131,13 @@ For "real-time" endpoints which are billed per hour of usage, specify `realtime`
 By default, the `azureaistudio` service sets the number of requests allowed per minute to `240`.
 This helps to minimize the number of rate limit errors returned from Azure AI Studio.
 To modify this, set the `requests_per_minute` setting of this object in your service settings:
-```
++
+[source,text]
+----
 "rate_limit": {
     "requests_per_minute": <<number_of_requests>>
 }
-```
+----
 =====
 +
 .`service_settings` for the `azureopenai` service
@@ -181,6 +172,22 @@ Your Azure OpenAI deployments can be found though the https://oai.azure.com/[Azu
 The Azure API version ID to use.
 We recommend using the https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#embeddings[latest supported non-preview version].
 
+`rate_limit`:::
+(Optional, object)
+The `azureopenai` service sets a default number of requests allowed per minute depending on the task type.
+For `text_embedding` it is set to `1440`.
+For `completion` it is set to `120`.
+This helps to minimize the number of rate limit errors returned from Azure.
+To modify this, set the `requests_per_minute` setting of this object in your service settings:
++
+[source,text]
+----
+"rate_limit": {
+    "requests_per_minute": <<number_of_requests>>
+}
+----
++
+More information about the rate limits for Azure can be found in the https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits[Quota limits docs] and https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/quota?tabs=rest[How to change the quotas].
 =====
 +
 .`service_settings` for the `cohere` service
@@ -188,24 +195,24 @@ We recommend using the https://learn.microsoft.com/en-us/azure/ai-services/opena
 =====
 `api_key`:::
 (Required, string)
-A valid API key of your Cohere account. You can find your Cohere API keys or you
-can create a new one
+A valid API key of your Cohere account.
+You can find your Cohere API keys or you can create a new one
 https://dashboard.cohere.com/api-keys[on the API keys settings page].
 
-IMPORTANT: You need to provide the API key only once, during the {infer} model
-creation. The <<get-inference-api>> does not retrieve your API key. After
-creating the {infer} model, you cannot change the associated API key. If you
-want to use a different API key, delete the {infer} model and recreate it with
-the same name and the updated API key.
+IMPORTANT: You need to provide the API key only once, during the {infer} model creation.
+The <<get-inference-api>> does not retrieve your API key.
+After creating the {infer} model, you cannot change the associated API key.
+If you want to use a different API key, delete the {infer} model and recreate it with the same name and the updated API key.
 
 `embedding_type`::
 (Optional, string)
-Only for `text_embedding`. Specifies the types of embeddings you want to get
-back. Defaults to `float`.
+Only for `text_embedding`.
+Specifies the types of embeddings you want to get back.
+Defaults to `float`.
 Valid values are:
-  * `byte`: use it for signed int8 embeddings (this is a synonym of `int8`).
-  * `float`: use it for the default float embeddings.
-  * `int8`: use it for signed int8 embeddings.
+* `byte`: use it for signed int8 embeddings (this is a synonym of `int8`).
+* `float`: use it for the default float embeddings.
+* `int8`: use it for signed int8 embeddings.
 
 `model_id`::
 (Optional, string)
@@ -214,50 +221,68 @@ To review the available `rerank` models, refer to the
 https://docs.cohere.com/reference/rerank-1[Cohere docs].
 
 To review the available `text_embedding` models, refer to the
-https://docs.cohere.com/reference/embed[Cohere docs]. The default value for
+https://docs.cohere.com/reference/embed[Cohere docs].
+The default value for
 `text_embedding` is `embed-english-v2.0`.
+
+`rate_limit`:::
+(Optional, object)
+By default, the `cohere` service sets the number of requests allowed per minute to `10000`.
+This value is the same for all task types.
+This helps to minimize the number of rate limit errors returned from Cohere.
+To modify this, set the `requests_per_minute` setting of this object in your service settings:
++
+[source,text]
+----
+"rate_limit": {
+    "requests_per_minute": <<number_of_requests>>
+}
+----
++
+More information about Cohere's rate limits can be found in https://docs.cohere.com/docs/going-live#production-key-specifications[Cohere's production key docs].
+
 =====
 +
 .`service_settings` for the `elasticsearch` service
 [%collapsible%closed]
 =====
+
 `model_id`:::
 (Required, string)
-The name of the model to use for the {infer} task. It can be the
-ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or
-a text embedding model already
+The name of the model to use for the {infer} task.
+It can be the ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or a text embedding model already
 {ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
 
 `num_allocations`:::
 (Required, integer)
-The number of model allocations to create. `num_allocations` must not exceed the
-number of available processors per node divided by the `num_threads`.
+The number of model allocations to create. `num_allocations` must not exceed the number of available processors per node divided by the `num_threads`.
 
 `num_threads`:::
 (Required, integer)
-The number of threads to use by each model allocation. `num_threads` must not
-exceed the number of available processors per node divided by the number of
-allocations. Must be a power of 2. Max allowed value is 32.
+The number of threads to use by each model allocation. `num_threads` must not exceed the number of available processors per node divided by the number of allocations.
+Must be a power of 2. Max allowed value is 32.
+
 =====
 +
 .`service_settings` for the `elser` service
 [%collapsible%closed]
 =====
+
 `num_allocations`:::
 (Required, integer)
-The number of model allocations to create. `num_allocations` must not exceed the
-number of available processors per node divided by the `num_threads`.
+The number of model allocations to create. `num_allocations` must not exceed the number of available processors per node divided by the `num_threads`.
 
 `num_threads`:::
 (Required, integer)
-The number of threads to use by each model allocation. `num_threads` must not
-exceed the number of available processors per node divided by the number of
-allocations. Must be a power of 2. Max allowed value is 32.
+The number of threads to use by each model allocation. `num_threads` must not exceed the number of available processors per node divided by the number of allocations.
+Must be a power of 2. Max allowed value is 32.
+
 =====
 +
 .`service_settings` for the `googleiastudio` service
 [%collapsible%closed]
 =====
+
 `api_key`:::
 (Required, string)
 A valid API key for the Google Gemini API.
@@ -274,76 +299,113 @@ This helps to minimize the number of rate limit errors returned from Google AI S
 To modify this, set the `requests_per_minute` setting of this object in your service settings:
 +
 --
-```
+[source,text]
+----
 "rate_limit": {
     "requests_per_minute": <<number_of_requests>>
 }
-```
+----
 --
+
 =====
 +
 .`service_settings` for the `hugging_face` service
 [%collapsible%closed]
 =====
+
 `api_key`:::
 (Required, string)
-A valid access token of your Hugging Face account. You can find your Hugging
-Face access tokens or you can create a new one
+A valid access token of your Hugging Face account.
+You can find your Hugging Face access tokens or you can create a new one
 https://huggingface.co/settings/tokens[on the settings page].
 
-IMPORTANT: You need to provide the API key only once, during the {infer} model
-creation. The <<get-inference-api>> does not retrieve your API key. After
-creating the {infer} model, you cannot change the associated API key. If you
-want to use a different API key, delete the {infer} model and recreate it with
-the same name and the updated API key.
+IMPORTANT: You need to provide the API key only once, during the {infer} model creation.
+The <<get-inference-api>> does not retrieve your API key.
+After creating the {infer} model, you cannot change the associated API key.
+If you want to use a different API key, delete the {infer} model and recreate it with the same name and the updated API key.
 
 `url`:::
 (Required, string)
 The URL endpoint to use for the requests.
+
+`rate_limit`:::
+(Optional, object)
+By default, the `huggingface` service sets the number of requests allowed per minute to `3000`.
+This helps to minimize the number of rate limit errors returned from Hugging Face.
+To modify this, set the `requests_per_minute` setting of this object in your service settings:
++
+[source,text]
+----
+"rate_limit": {
+    "requests_per_minute": <<number_of_requests>>
+}
+----
+
 =====
 +
 .`service_settings` for the `openai` service
 [%collapsible%closed]
 =====
+
 `api_key`:::
 (Required, string)
-A valid API key of your OpenAI account. You can find your OpenAI API keys in
-your OpenAI account under the
+A valid API key of your OpenAI account.
+You can find your OpenAI API keys in your OpenAI account under the
 https://platform.openai.com/api-keys[API keys section].
 
-IMPORTANT: You need to provide the API key only once, during the {infer} model
-creation. The <<get-inference-api>> does not retrieve your API key. After
-creating the {infer} model, you cannot change the associated API key. If you
-want to use a different API key, delete the {infer} model and recreate it with
-the same name and the updated API key.
+IMPORTANT: You need to provide the API key only once, during the {infer} model creation.
+The <<get-inference-api>> does not retrieve your API key.
+After creating the {infer} model, you cannot change the associated API key.
+If you want to use a different API key, delete the {infer} model and recreate it with the same name and the updated API key.
 
 `model_id`:::
 (Required, string)
-The name of the model to use for the {infer} task. Refer to the
+The name of the model to use for the {infer} task.
+Refer to the
 https://platform.openai.com/docs/guides/embeddings/what-are-embeddings[OpenAI documentation]
 for the list of available text embedding models.
 
 `organization_id`:::
 (Optional, string)
-The unique identifier of your organization. You can find the Organization ID in
-your OpenAI account under
+The unique identifier of your organization.
+You can find the Organization ID in your OpenAI account under
 https://platform.openai.com/account/organization[**Settings** > **Organizations**].
 
 `url`:::
 (Optional, string)
-The URL endpoint to use for the requests. Can be changed for testing purposes.
+The URL endpoint to use for the requests.
+Can be changed for testing purposes.
 Defaults to `https://api.openai.com/v1/embeddings`.
 
+`rate_limit`:::
+(Optional, object)
+The `openai` service sets a default number of requests allowed per minute depending on the task type.
+For `text_embedding` it is set to `3000`.
+For `completion` it is set to `500`.
+This helps to minimize the number of rate limit errors returned from Azure.
+To modify this, set the `requests_per_minute` setting of this object in your service settings:
++
+[source,text]
+----
+"rate_limit": {
+    "requests_per_minute": <<number_of_requests>>
+}
+----
++
+More information about the rate limits for OpenAI can be found in your https://platform.openai.com/account/limits[Account limits].
+
 =====
 
 `task_settings`::
 (Optional, object)
-Settings to configure the {infer} task. These settings are specific to the
+Settings to configure the {infer} task.
+These settings are specific to the
 `<task_type>` you specified.
 +
 .`task_settings` for the `completion` task type
 [%collapsible%closed]
 =====
+
 `do_sample`:::
 (Optional, float)
 For the `azureaistudio` service only.
@@ -358,8 +420,8 @@ Defaults to 64.
 
 `user`:::
 (Optional, string)
-For `openai` service only. Specifies the user issuing the request, which can be
-used for abuse detection.
+For `openai` service only.
+Specifies the user issuing the request, which can be used for abuse detection.
 
 `temperature`:::
 (Optional, float)
@@ -378,45 +440,46 @@ Should not be used if `temperature` is specified.
 .`task_settings` for the `rerank` task type
 [%collapsible%closed]
 =====
+
 `return_documents`::
 (Optional, boolean)
-For `cohere` service only. Specify whether to return doc text within the
-results.
+For `cohere` service only.
+Specify whether to return doc text within the results.
 
 `top_n`::
 (Optional, integer)
-The number of most relevant documents to return, defaults to the number of the
-documents.
+The number of most relevant documents to return, defaults to the number of the documents.
+
 =====
 +
 .`task_settings` for the `text_embedding` task type
 [%collapsible%closed]
 =====
+
 `input_type`:::
 (Optional, string)
-For `cohere` service only. Specifies the type of input passed to the model.
+For `cohere` service only.
+Specifies the type of input passed to the model.
 Valid values are:
-  * `classification`: use it for embeddings passed through a text classifier.
-  * `clusterning`: use it for the embeddings run through a clustering algorithm.
-  * `ingest`: use it for storing document embeddings in a vector database.
-  * `search`: use it for storing embeddings of search queries run against a
-  vector database to find relevant documents.
+* `classification`: use it for embeddings passed through a text classifier.
+* `clusterning`: use it for the embeddings run through a clustering algorithm.
+* `ingest`: use it for storing document embeddings in a vector database.
+* `search`: use it for storing embeddings of search queries run against a vector database to find relevant documents.
 
 `truncate`:::
 (Optional, string)
-For `cohere` service only. Specifies how the API handles inputs longer than the
-maximum token length. Defaults to `END`. Valid values are:
- * `NONE`: when the input exceeds the maximum input token length an error is
- returned.
- * `START`: when the input exceeds the maximum input token length the start of
- the input is discarded.
- * `END`: when the input exceeds the maximum input token length the end of
- the input is discarded.
+For `cohere` service only.
+Specifies how the API handles inputs longer than the maximum token length.
+Defaults to `END`.
+Valid values are:
+* `NONE`: when the input exceeds the maximum input token length an error is returned.
+* `START`: when the input exceeds the maximum input token length the start of the input is discarded.
+* `END`: when the input exceeds the maximum input token length the end of the input is discarded.
 
 `user`:::
 (optional, string)
-For `openai`, `azureopenai` and `azureaistudio` services only. Specifies the user issuing the
-request, which can be used for abuse detection.
+For `openai`, `azureopenai` and `azureaistudio` services only.
+Specifies the user issuing the request, which can be used for abuse detection.
 
 =====
 [discrete]
@@ -470,7 +533,6 @@ PUT _inference/completion/azure_ai_studio_completion
 
 The list of chat completion models that you can choose from in your deployment can be found in the https://ai.azure.com/explore/models?selectedTask=chat-completion[Azure AI Studio model explorer].
 
-
 [discrete]
 [[inference-example-azureopenai]]
 ===== Azure OpenAI service
@@ -519,7 +581,6 @@ The list of chat completion models that you can choose from in your Azure OpenAI
 * https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-4-and-gpt-4-turbo-models[GPT-4 and GPT-4 Turbo models]
 * https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-35[GPT-3.5]
 
-
 [discrete]
 [[inference-example-cohere]]
 ===== Cohere service
@@ -565,7 +626,6 @@ PUT _inference/rerank/cohere-rerank
 For more examples, also review the
 https://docs.cohere.com/docs/elasticsearch-and-cohere#rerank-search-results-with-cohere-and-elasticsearch[Cohere documentation].
 
-
 [discrete]
 [[inference-example-e5]]
 ===== E5 via the `elasticsearch` service
@@ -586,10 +646,9 @@ PUT _inference/text_embedding/my-e5-model
 }
 ------------------------------------------------------------
 // TEST[skip:TBD]
-<1> The `model_id` must be the ID of one of the built-in E5 models. Valid values
-are `.multilingual-e5-small` and `.multilingual-e5-small_linux-x86_64`. For
-further details, refer to the {ml-docs}/ml-nlp-e5.html[E5 model documentation].
-
+<1> The `model_id` must be the ID of one of the built-in E5 models.
+Valid values are `.multilingual-e5-small` and `.multilingual-e5-small_linux-x86_64`.
+For further details, refer to the {ml-docs}/ml-nlp-e5.html[E5 model documentation].
 
 [discrete]
 [[inference-example-elser]]
@@ -597,8 +656,7 @@ further details, refer to the {ml-docs}/ml-nlp-e5.html[E5 model documentation].
 
 The following example shows how to create an {infer} endpoint called
 `my-elser-model` to perform a `sparse_embedding` task type.
-Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more
-info.
+Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more info.
 
 [source,console]
 ------------------------------------------------------------
@@ -672,16 +730,17 @@ PUT _inference/text_embedding/hugging-face-embeddings
 }
 ------------------------------------------------------------
 // TEST[skip:TBD]
-<1> A valid Hugging Face access token. You can find on the
+<1> A valid Hugging Face access token.
+You can find on the
 https://huggingface.co/settings/tokens[settings page of your account].
 <2> The {infer} endpoint URL you created on Hugging Face.
 
 Create a new {infer} endpoint on
-https://ui.endpoints.huggingface.co/[the Hugging Face endpoint page] to get an
-endpoint URL. Select the model you want to use on the new endpoint creation page
-- for example `intfloat/e5-small-v2` - then select the `Sentence Embeddings`
-task under the Advanced configuration section. Create the endpoint. Copy the URL
-after the endpoint initialization has been finished.
+https://ui.endpoints.huggingface.co/[the Hugging Face endpoint page] to get an endpoint URL.
+Select the model you want to use on the new endpoint creation page - for example `intfloat/e5-small-v2` - then select the `Sentence Embeddings`
+task under the Advanced configuration section.
+Create the endpoint.
+Copy the URL after the endpoint initialization has been finished.
 
 [discrete]
 [[inference-example-hugging-face-supported-models]]
@@ -695,7 +754,6 @@ The list of recommended models for the Hugging Face service:
 * https://huggingface.co/intfloat/multilingual-e5-base[multilingual-e5-base]
 * https://huggingface.co/intfloat/multilingual-e5-small[multilingual-e5-small]
 
-
 [discrete]
 [[inference-example-eland]]
 ===== Models uploaded by Eland via the elasticsearch service
@@ -716,11 +774,9 @@ PUT _inference/text_embedding/my-msmarco-minilm-model
 }
 ------------------------------------------------------------
 // TEST[skip:TBD]
-<1> The `model_id` must be the ID of a text embedding model which has already
-been
+<1> The `model_id` must be the ID of a text embedding model which has already been
 {ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
 
-
 [discrete]
 [[inference-example-openai]]
 ===== OpenAI service
@@ -756,4 +812,3 @@ PUT _inference/completion/openai-completion
 }
 ------------------------------------------------------------
 // TEST[skip:TBD]
-
diff --git a/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java b/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java
index df7c47943289d..26d93bca6b09a 100644
--- a/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java
+++ b/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java
@@ -88,6 +88,13 @@ public static TimeValue timeValueDays(long days) {
         return new TimeValue(days, TimeUnit.DAYS);
     }
 
+    /**
+     * @return the {@link TimeValue} object that has the least duration.
+     */
+    public static TimeValue min(TimeValue time1, TimeValue time2) {
+        return time1.compareTo(time2) < 0 ? time1 : time2;
+    }
+
     /**
      * @return the unit used for the this time value, see {@link #duration()}
      */
diff --git a/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
index b6481db9b9951..dd2755ac1f9f7 100644
--- a/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
+++ b/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
@@ -17,6 +17,7 @@
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.object.HasToString.hasToString;
 
@@ -231,6 +232,12 @@ public void testRejectsNegativeValuesAtCreation() {
         assertThat(ex.getMessage(), containsString("duration cannot be negative"));
     }
 
+    public void testMin() {
+        assertThat(TimeValue.min(TimeValue.ZERO, TimeValue.timeValueNanos(1)), is(TimeValue.timeValueNanos(0)));
+        assertThat(TimeValue.min(TimeValue.MAX_VALUE, TimeValue.timeValueNanos(1)), is(TimeValue.timeValueNanos(1)));
+        assertThat(TimeValue.min(TimeValue.MINUS_ONE, TimeValue.timeValueHours(1)), is(TimeValue.MINUS_ONE));
+    }
+
     private TimeUnit randomTimeUnitObject() {
         return randomFrom(
             TimeUnit.NANOSECONDS,
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java
index 140c08ceef80f..81bc90433d34a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java
@@ -26,6 +26,7 @@ public class CohereActionCreator implements CohereActionVisitor {
     private final ServiceComponents serviceComponents;
 
     public CohereActionCreator(Sender sender, ServiceComponents serviceComponents) {
+        // TODO Batching - accept a class that can handle batching
         this.sender = Objects.requireNonNull(sender);
         this.serviceComponents = Objects.requireNonNull(serviceComponents);
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java
index 63e51d99a8cee..b4815f8f0d1bf 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java
@@ -36,6 +36,7 @@ public CohereEmbeddingsAction(Sender sender, CohereEmbeddingsModel model, Thread
             model.getServiceSettings().getCommonSettings().uri(),
             "Cohere embeddings"
         );
+        // TODO - Batching pass the batching class on to the CohereEmbeddingsRequestManager
         requestCreator = CohereEmbeddingsRequestManager.of(model, threadPool);
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java
index deff410aebaa8..002fa71b7fb5d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioChatCompletionRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -37,17 +36,16 @@ public AzureAiStudioChatCompletionRequestManager(AzureAiStudioChatCompletionMode
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         AzureAiStudioChatCompletionRequest request = new AzureAiStudioChatCompletionRequest(model, input);
 
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 
     private static ResponseHandler createCompletionHandler() {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioEmbeddingsRequestManager.java
index a2b363151a417..ec5ab2fee6a57 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioEmbeddingsRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureAiStudioEmbeddingsRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -41,17 +40,16 @@ public AzureAiStudioEmbeddingsRequestManager(AzureAiStudioEmbeddingsModel model,
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         var truncatedInput = truncate(input, model.getServiceSettings().maxInputTokens());
         AzureAiStudioEmbeddingsRequest request = new AzureAiStudioEmbeddingsRequest(truncator, truncatedInput, model);
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 
     private static ResponseHandler createEmbeddingsHandler() {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiCompletionRequestManager.java
index 2811155f6f357..5206d6c2c23cc 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiCompletionRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiCompletionRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -43,16 +42,15 @@ public AzureOpenAiCompletionRequestManager(AzureOpenAiCompletionModel model, Thr
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         @Nullable String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         AzureOpenAiCompletionRequest request = new AzureOpenAiCompletionRequest(input, model);
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiEmbeddingsRequestManager.java
index 06152b50822aa..e0fcee30e5af3 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiEmbeddingsRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiEmbeddingsRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -55,16 +54,15 @@ public AzureOpenAiEmbeddingsRequestManager(AzureOpenAiEmbeddingsModel model, Tru
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         var truncatedInput = truncate(input, model.getServiceSettings().maxInputTokens());
         AzureOpenAiEmbeddingsRequest request = new AzureOpenAiEmbeddingsRequest(truncator, truncatedInput, model);
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManager.java
index abca0ce0d049b..a015716b81032 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManager.java
@@ -38,11 +38,16 @@ public String inferenceEntityId() {
 
     @Override
     public Object rateLimitGrouping() {
-        return rateLimitGroup;
+        // It's possible that two inference endpoints have the same information defining the group but have different
+        // rate limits then they should be in different groups otherwise whoever initially created the group will set
+        // the rate and the other inference endpoint's rate will be ignored
+        return new EndpointGrouping(rateLimitGroup, rateLimitSettings);
     }
 
     @Override
     public RateLimitSettings rateLimitSettings() {
         return rateLimitSettings;
     }
+
+    private record EndpointGrouping(Object group, RateLimitSettings settings) {}
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java
index 255d4a3f3879f..8a4b0e45b93fa 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -46,16 +45,15 @@ private CohereCompletionRequestManager(CohereCompletionModel model, ThreadPool t
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         CohereCompletionRequest request = new CohereCompletionRequest(input, model);
 
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java
index 0bf1c11285adb..a51910f1d0a67 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -44,16 +43,15 @@ private CohereEmbeddingsRequestManager(CohereEmbeddingsModel model, ThreadPool t
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         CohereEmbeddingsRequest request = new CohereEmbeddingsRequest(input, model);
 
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java
index 1778663a194e8..1351eec406569 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -44,16 +43,15 @@ private CohereRerankRequestManager(CohereRerankModel model, ThreadPool threadPoo
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         CohereRerankRequest request = new CohereRerankRequest(query, input, model);
 
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java
index 53f30773cbfe3..214eba4ee3485 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java
@@ -23,7 +23,6 @@ record ExecutableInferenceRequest(
     RequestSender requestSender,
     Logger logger,
     Request request,
-    HttpClientContext context,
     ResponseHandler responseHandler,
     Supplier<Boolean> hasFinished,
     ActionListener<InferenceServiceResults> listener
@@ -34,7 +33,7 @@ public void run() {
         var inferenceEntityId = request.createHttpRequest().inferenceEntityId();
 
         try {
-            requestSender.send(logger, request, context, hasFinished, responseHandler, listener);
+            requestSender.send(logger, request, HttpClientContext.create(), hasFinished, responseHandler, listener);
         } catch (Exception e) {
             var errorMessage = Strings.format("Failed to send request from inference entity id [%s]", inferenceEntityId);
             logger.warn(errorMessage, e);
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java
index eb9baa680446a..2b191b046477b 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioCompletionRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -42,15 +41,14 @@ public GoogleAiStudioCompletionRequestManager(GoogleAiStudioCompletionModel mode
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         GoogleAiStudioCompletionRequest request = new GoogleAiStudioCompletionRequest(input, model);
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioEmbeddingsRequestManager.java
index 15c2825e7d043..6436e0231ab48 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioEmbeddingsRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/GoogleAiStudioEmbeddingsRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -48,17 +47,16 @@ public GoogleAiStudioEmbeddingsRequestManager(GoogleAiStudioEmbeddingsModel mode
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         var truncatedInput = truncate(input, model.getServiceSettings().maxInputTokens());
         GoogleAiStudioEmbeddingsRequest request = new GoogleAiStudioEmbeddingsRequest(truncator, truncatedInput, model);
 
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java
index 21a758a3db248..d1e309a774ab7 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java
@@ -15,6 +15,8 @@
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.inference.external.http.HttpClientManager;
+import org.elasticsearch.xpack.inference.external.http.RequestExecutor;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
 import org.elasticsearch.xpack.inference.external.http.retry.RetrySettings;
 import org.elasticsearch.xpack.inference.external.http.retry.RetryingHttpSender;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
@@ -39,30 +41,28 @@ public static class Factory {
         private final ServiceComponents serviceComponents;
         private final HttpClientManager httpClientManager;
         private final ClusterService clusterService;
-        private final SingleRequestManager requestManager;
+        private final RequestSender requestSender;
 
         public Factory(ServiceComponents serviceComponents, HttpClientManager httpClientManager, ClusterService clusterService) {
             this.serviceComponents = Objects.requireNonNull(serviceComponents);
             this.httpClientManager = Objects.requireNonNull(httpClientManager);
             this.clusterService = Objects.requireNonNull(clusterService);
 
-            var requestSender = new RetryingHttpSender(
+            requestSender = new RetryingHttpSender(
                 this.httpClientManager.getHttpClient(),
                 serviceComponents.throttlerManager(),
                 new RetrySettings(serviceComponents.settings(), clusterService),
                 serviceComponents.threadPool()
             );
-            requestManager = new SingleRequestManager(requestSender);
         }
 
-        public Sender createSender(String serviceName) {
+        public Sender createSender() {
             return new HttpRequestSender(
-                serviceName,
                 serviceComponents.threadPool(),
                 httpClientManager,
                 clusterService,
                 serviceComponents.settings(),
-                requestManager
+                requestSender
             );
         }
     }
@@ -71,26 +71,24 @@ public Sender createSender(String serviceName) {
 
     private final ThreadPool threadPool;
     private final HttpClientManager manager;
-    private final RequestExecutorService service;
+    private final RequestExecutor service;
     private final AtomicBoolean started = new AtomicBoolean(false);
     private final CountDownLatch startCompleted = new CountDownLatch(1);
 
     private HttpRequestSender(
-        String serviceName,
         ThreadPool threadPool,
         HttpClientManager httpClientManager,
         ClusterService clusterService,
         Settings settings,
-        SingleRequestManager requestManager
+        RequestSender requestSender
     ) {
         this.threadPool = Objects.requireNonNull(threadPool);
         this.manager = Objects.requireNonNull(httpClientManager);
         service = new RequestExecutorService(
-            serviceName,
             threadPool,
             startCompleted,
             new RequestExecutorServiceSettings(settings, clusterService),
-            requestManager
+            requestSender
         );
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HuggingFaceRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HuggingFaceRequestManager.java
index 7c09e0c67c1c6..6c8fc446d5243 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HuggingFaceRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HuggingFaceRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -55,26 +54,17 @@ private HuggingFaceRequestManager(HuggingFaceModel model, ResponseHandler respon
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         var truncatedInput = truncate(input, model.getTokenLimit());
         var request = new HuggingFaceInferenceRequest(truncator, truncatedInput, model);
 
-        return new ExecutableInferenceRequest(
-            requestSender,
-            logger,
-            request,
-            context,
-            responseHandler,
-            hasRequestCompletedFunction,
-            listener
-        );
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, responseHandler, hasRequestCompletedFunction, listener));
     }
 
     record RateLimitGrouping(int accountHash) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceRequest.java
index 3c711bb79717c..6199a75a41a7d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceRequest.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceRequest.java
@@ -19,9 +19,9 @@
 public interface InferenceRequest {
 
     /**
-     * Returns the creator that handles building an executable request based on the input provided.
+     * Returns the manager that handles building and executing an inference request.
      */
-    RequestManager getRequestCreator();
+    RequestManager getRequestManager();
 
     /**
      * Returns the query associated with this request. Used for Rerank tasks.
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/MistralEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/MistralEmbeddingsRequestManager.java
index f31a633581705..ab6a1bfb31372 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/MistralEmbeddingsRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/MistralEmbeddingsRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -51,18 +50,17 @@ public MistralEmbeddingsRequestManager(MistralEmbeddingsModel model, Truncator t
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         var truncatedInput = truncate(input, model.getServiceSettings().maxInputTokens());
         MistralEmbeddingsRequest request = new MistralEmbeddingsRequest(truncator, truncatedInput, model);
 
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 
     record RateLimitGrouping(int keyHashCode) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java
deleted file mode 100644
index 0355880b3f714..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.http.sender;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.inference.InferenceServiceResults;
-
-import java.util.List;
-import java.util.function.Supplier;
-
-class NoopTask implements RejectableTask {
-
-    @Override
-    public RequestManager getRequestCreator() {
-        return null;
-    }
-
-    @Override
-    public String getQuery() {
-        return null;
-    }
-
-    @Override
-    public List<String> getInput() {
-        return null;
-    }
-
-    @Override
-    public ActionListener<InferenceServiceResults> getListener() {
-        return null;
-    }
-
-    @Override
-    public boolean hasCompleted() {
-        return true;
-    }
-
-    @Override
-    public Supplier<Boolean> getRequestCompletedFunction() {
-        return () -> true;
-    }
-
-    @Override
-    public void onRejection(Exception e) {
-
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java
index 9c6c216c61272..7bc09fd76736b 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -43,17 +42,16 @@ private OpenAiCompletionRequestManager(OpenAiChatCompletionModel model, ThreadPo
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         @Nullable String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         OpenAiChatCompletionRequest request = new OpenAiChatCompletionRequest(input, model);
 
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 
     private static ResponseHandler createCompletionHandler() {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java
index 3a0a8fd64a656..41f91d2b89ee5 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -55,17 +54,16 @@ private OpenAiEmbeddingsRequestManager(OpenAiEmbeddingsModel model, Truncator tr
     }
 
     @Override
-    public Runnable create(
+    public void execute(
         String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     ) {
         var truncatedInput = truncate(input, model.getServiceSettings().maxInputTokens());
         OpenAiEmbeddingsRequest request = new OpenAiEmbeddingsRequest(truncator, truncatedInput, model);
 
-        return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener);
+        execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener));
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java
index d5a13c2e0771d..38d47aec68eb6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -17,21 +16,31 @@
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.inference.common.AdjustableCapacityBlockingQueue;
+import org.elasticsearch.xpack.inference.common.RateLimiter;
 import org.elasticsearch.xpack.inference.external.http.RequestExecutor;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
+import java.time.Clock;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Consumer;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME;
 
 /**
  * A service for queuing and executing {@link RequestTask}. This class is useful because the
@@ -45,7 +54,18 @@
  * {@link org.apache.http.client.config.RequestConfig.Builder#setConnectionRequestTimeout} for more info.
  */
 class RequestExecutorService implements RequestExecutor {
-    private static final AdjustableCapacityBlockingQueue.QueueCreator<RejectableTask> QUEUE_CREATOR =
+
+    /**
+     * Provides dependency injection mainly for testing
+     */
+    interface Sleeper {
+        void sleep(TimeValue sleepTime) throws InterruptedException;
+    }
+
+    // default for tests
+    static final Sleeper DEFAULT_SLEEPER = sleepTime -> sleepTime.timeUnit().sleep(sleepTime.duration());
+    // default for tests
+    static final AdjustableCapacityBlockingQueue.QueueCreator<RejectableTask> DEFAULT_QUEUE_CREATOR =
         new AdjustableCapacityBlockingQueue.QueueCreator<>() {
             @Override
             public BlockingQueue<RejectableTask> create(int capacity) {
@@ -65,86 +85,116 @@ public BlockingQueue<RejectableTask> create() {
             }
         };
 
+    /**
+     * Provides dependency injection mainly for testing
+     */
+    interface RateLimiterCreator {
+        RateLimiter create(double accumulatedTokensLimit, double tokensPerTimeUnit, TimeUnit unit);
+    }
+
+    // default for testing
+    static final RateLimiterCreator DEFAULT_RATE_LIMIT_CREATOR = RateLimiter::new;
     private static final Logger logger = LogManager.getLogger(RequestExecutorService.class);
-    private final String serviceName;
-    private final AdjustableCapacityBlockingQueue<RejectableTask> queue;
-    private final AtomicBoolean running = new AtomicBoolean(true);
-    private final CountDownLatch terminationLatch = new CountDownLatch(1);
-    private final HttpClientContext httpContext;
+    private static final TimeValue RATE_LIMIT_GROUP_CLEANUP_INTERVAL = TimeValue.timeValueDays(1);
+
+    private final ConcurrentMap<Object, RateLimitingEndpointHandler> rateLimitGroupings = new ConcurrentHashMap<>();
     private final ThreadPool threadPool;
     private final CountDownLatch startupLatch;
-    private final BlockingQueue<Runnable> controlQueue = new LinkedBlockingQueue<>();
-    private final SingleRequestManager requestManager;
+    private final CountDownLatch terminationLatch = new CountDownLatch(1);
+    private final RequestSender requestSender;
+    private final RequestExecutorServiceSettings settings;
+    private final Clock clock;
+    private final AtomicBoolean shutdown = new AtomicBoolean(false);
+    private final AdjustableCapacityBlockingQueue.QueueCreator<RejectableTask> queueCreator;
+    private final Sleeper sleeper;
+    private final RateLimiterCreator rateLimiterCreator;
+    private final AtomicReference<Scheduler.Cancellable> cancellableCleanupTask = new AtomicReference<>();
+    private final AtomicBoolean started = new AtomicBoolean(false);
 
     RequestExecutorService(
-        String serviceName,
         ThreadPool threadPool,
         @Nullable CountDownLatch startupLatch,
         RequestExecutorServiceSettings settings,
-        SingleRequestManager requestManager
+        RequestSender requestSender
     ) {
-        this(serviceName, threadPool, QUEUE_CREATOR, startupLatch, settings, requestManager);
+        this(
+            threadPool,
+            DEFAULT_QUEUE_CREATOR,
+            startupLatch,
+            settings,
+            requestSender,
+            Clock.systemUTC(),
+            DEFAULT_SLEEPER,
+            DEFAULT_RATE_LIMIT_CREATOR
+        );
     }
 
-    /**
-     * This constructor should only be used directly for testing.
-     */
     RequestExecutorService(
-        String serviceName,
         ThreadPool threadPool,
-        AdjustableCapacityBlockingQueue.QueueCreator<RejectableTask> createQueue,
+        AdjustableCapacityBlockingQueue.QueueCreator<RejectableTask> queueCreator,
         @Nullable CountDownLatch startupLatch,
         RequestExecutorServiceSettings settings,
-        SingleRequestManager requestManager
+        RequestSender requestSender,
+        Clock clock,
+        Sleeper sleeper,
+        RateLimiterCreator rateLimiterCreator
     ) {
-        this.serviceName = Objects.requireNonNull(serviceName);
         this.threadPool = Objects.requireNonNull(threadPool);
-        this.httpContext = HttpClientContext.create();
-        this.queue = new AdjustableCapacityBlockingQueue<>(createQueue, settings.getQueueCapacity());
+        this.queueCreator = Objects.requireNonNull(queueCreator);
         this.startupLatch = startupLatch;
-        this.requestManager = Objects.requireNonNull(requestManager);
+        this.requestSender = Objects.requireNonNull(requestSender);
+        this.settings = Objects.requireNonNull(settings);
+        this.clock = Objects.requireNonNull(clock);
+        this.sleeper = Objects.requireNonNull(sleeper);
+        this.rateLimiterCreator = Objects.requireNonNull(rateLimiterCreator);
+    }
 
-        Objects.requireNonNull(settings);
-        settings.registerQueueCapacityCallback(this::onCapacityChange);
+    public void shutdown() {
+        if (shutdown.compareAndSet(false, true)) {
+            if (cancellableCleanupTask.get() != null) {
+                logger.debug(() -> "Stopping clean up thread");
+                cancellableCleanupTask.get().cancel();
+            }
+        }
     }
 
-    private void onCapacityChange(int capacity) {
-        logger.debug(() -> Strings.format("Setting queue capacity to [%s]", capacity));
+    public boolean isShutdown() {
+        return shutdown.get();
+    }
 
-        var enqueuedCapacityCommand = controlQueue.offer(() -> updateCapacity(capacity));
-        if (enqueuedCapacityCommand == false) {
-            logger.warn("Failed to change request batching service queue capacity. Control queue was full, please try again later.");
-        } else {
-            // ensure that the task execution loop wakes up
-            queue.offer(new NoopTask());
-        }
+    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+        return terminationLatch.await(timeout, unit);
     }
 
-    private void updateCapacity(int newCapacity) {
-        try {
-            queue.setCapacity(newCapacity);
-        } catch (Exception e) {
-            logger.warn(
-                format("Failed to set the capacity of the task queue to [%s] for request batching service [%s]", newCapacity, serviceName),
-                e
-            );
-        }
+    public boolean isTerminated() {
+        return terminationLatch.getCount() == 0;
+    }
+
+    public int queueSize() {
+        return rateLimitGroupings.values().stream().mapToInt(RateLimitingEndpointHandler::queueSize).sum();
     }
 
     /**
      * Begin servicing tasks.
+     * <p>
+     * <b>Note: This should only be called once for the life of the object.</b>
+     * </p>
      */
     public void start() {
         try {
+            assert started.get() == false : "start() can only be called once";
+            started.set(true);
+
+            startCleanupTask();
             signalStartInitiated();
 
-            while (running.get()) {
+            while (isShutdown() == false) {
                 handleTasks();
             }
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
         } finally {
-            running.set(false);
+            shutdown();
             notifyRequestsOfShutdown();
             terminationLatch.countDown();
         }
@@ -156,108 +206,68 @@ private void signalStartInitiated() {
         }
     }
 
-    /**
-     * Protects the task retrieval logic from an unexpected exception.
-     *
-     * @throws InterruptedException rethrows the exception if it occurred retrieving a task because the thread is likely attempting to
-     *                              shut down
-     */
-    private void handleTasks() throws InterruptedException {
-        try {
-            RejectableTask task = queue.take();
+    private void startCleanupTask() {
+        assert cancellableCleanupTask.get() == null : "The clean up task can only be set once";
+        cancellableCleanupTask.set(startCleanupThread(RATE_LIMIT_GROUP_CLEANUP_INTERVAL));
+    }
 
-            var command = controlQueue.poll();
-            if (command != null) {
-                command.run();
-            }
+    private Scheduler.Cancellable startCleanupThread(TimeValue interval) {
+        logger.debug(() -> Strings.format("Clean up task scheduled with interval [%s]", interval));
 
-            // TODO add logic to complete pending items in the queue before shutting down
-            if (running.get() == false) {
-                logger.debug(() -> format("Http executor service [%s] exiting", serviceName));
-                rejectTaskBecauseOfShutdown(task);
-            } else {
-                executeTask(task);
-            }
-        } catch (InterruptedException e) {
-            throw e;
-        } catch (Exception e) {
-            logger.warn(format("Http executor service [%s] failed while retrieving task for execution", serviceName), e);
-        }
+        return threadPool.scheduleWithFixedDelay(this::removeStaleGroupings, interval, threadPool.executor(UTILITY_THREAD_POOL_NAME));
     }
 
-    private void executeTask(RejectableTask task) {
-        try {
-            requestManager.execute(task, httpContext);
-        } catch (Exception e) {
-            logger.warn(format("Http executor service [%s] failed to execute request [%s]", serviceName, task), e);
+    // default for testing
+    void removeStaleGroupings() {
+        var now = Instant.now(clock);
+        for (var iter = rateLimitGroupings.values().iterator(); iter.hasNext();) {
+            var endpoint = iter.next();
+
+            // if the current time is after the last time the endpoint enqueued a request + allowed stale period then we'll remove it
+            if (now.isAfter(endpoint.timeOfLastEnqueue().plus(settings.getRateLimitGroupStaleDuration()))) {
+                endpoint.close();
+                iter.remove();
+            }
         }
     }
 
-    private synchronized void notifyRequestsOfShutdown() {
-        assert isShutdown() : "Requests should only be notified if the executor is shutting down";
-
-        try {
-            List<RejectableTask> notExecuted = new ArrayList<>();
-            queue.drainTo(notExecuted);
-
-            rejectTasks(notExecuted, this::rejectTaskBecauseOfShutdown);
-        } catch (Exception e) {
-            logger.warn(format("Failed to notify tasks of queuing service [%s] shutdown", serviceName));
+    private void handleTasks() throws InterruptedException {
+        var timeToWait = settings.getTaskPollFrequency();
+        for (var endpoint : rateLimitGroupings.values()) {
+            timeToWait = TimeValue.min(endpoint.executeEnqueuedTask(), timeToWait);
         }
-    }
 
-    private void rejectTaskBecauseOfShutdown(RejectableTask task) {
-        try {
-            task.onRejection(
-                new EsRejectedExecutionException(
-                    format("Failed to send request, queue service [%s] has shutdown prior to executing request", serviceName),
-                    true
-                )
-            );
-        } catch (Exception e) {
-            logger.warn(
-                format("Failed to notify request [%s] for service [%s] of rejection after queuing service shutdown", task, serviceName)
-            );
-        }
+        sleeper.sleep(timeToWait);
     }
 
-    private void rejectTasks(List<RejectableTask> tasks, Consumer<RejectableTask> rejectionFunction) {
-        for (var task : tasks) {
-            rejectionFunction.accept(task);
+    private void notifyRequestsOfShutdown() {
+        assert isShutdown() : "Requests should only be notified if the executor is shutting down";
+
+        for (var endpoint : rateLimitGroupings.values()) {
+            endpoint.notifyRequestsOfShutdown();
         }
     }
 
-    public int queueSize() {
-        return queue.size();
-    }
+    // default for testing
+    Integer remainingQueueCapacity(RequestManager requestManager) {
+        var endpoint = rateLimitGroupings.get(requestManager.rateLimitGrouping());
 
-    @Override
-    public void shutdown() {
-        if (running.compareAndSet(true, false)) {
-            // if this fails because the queue is full, that's ok, we just want to ensure that queue.take() returns
-            queue.offer(new NoopTask());
+        if (endpoint == null) {
+            return null;
         }
-    }
 
-    @Override
-    public boolean isShutdown() {
-        return running.get() == false;
-    }
-
-    @Override
-    public boolean isTerminated() {
-        return terminationLatch.getCount() == 0;
+        return endpoint.remainingCapacity();
     }
 
-    @Override
-    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
-        return terminationLatch.await(timeout, unit);
+    // default for testing
+    int numberOfRateLimitGroups() {
+        return rateLimitGroupings.size();
     }
 
     /**
      * Execute the request at some point in the future.
      *
-     * @param requestCreator the http request to send
+     * @param requestManager the http request to send
      * @param inferenceInputs the inputs to send in the request
      * @param timeout the maximum time to wait for this request to complete (failing or succeeding). Once the time elapses, the
      *                listener::onFailure is called with a {@link org.elasticsearch.ElasticsearchTimeoutException}.
@@ -265,13 +275,13 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE
      * @param listener an {@link ActionListener<InferenceServiceResults>} for the response or failure
      */
     public void execute(
-        RequestManager requestCreator,
+        RequestManager requestManager,
         InferenceInputs inferenceInputs,
         @Nullable TimeValue timeout,
         ActionListener<InferenceServiceResults> listener
     ) {
         var task = new RequestTask(
-            requestCreator,
+            requestManager,
             inferenceInputs,
             timeout,
             threadPool,
@@ -280,38 +290,230 @@ public void execute(
             ContextPreservingActionListener.wrapPreservingContext(listener, threadPool.getThreadContext())
         );
 
-        completeExecution(task);
+        var endpoint = rateLimitGroupings.computeIfAbsent(requestManager.rateLimitGrouping(), key -> {
+            var endpointHandler = new RateLimitingEndpointHandler(
+                Integer.toString(requestManager.rateLimitGrouping().hashCode()),
+                queueCreator,
+                settings,
+                requestSender,
+                clock,
+                requestManager.rateLimitSettings(),
+                this::isShutdown,
+                rateLimiterCreator
+            );
+
+            endpointHandler.init();
+            return endpointHandler;
+        });
+
+        endpoint.enqueue(task);
     }
 
-    private void completeExecution(RequestTask task) {
-        if (isShutdown()) {
-            EsRejectedExecutionException rejected = new EsRejectedExecutionException(
-                format("Failed to enqueue task because the http executor service [%s] has already shutdown", serviceName),
-                true
+    /**
+     * Provides rate limiting functionality for requests. A single {@link RateLimitingEndpointHandler} governs a group of requests.
+     * This allows many requests to be serialized if they are being sent too fast. If the rate limit has not been met they will be sent
+     * as soon as a thread is available.
+     */
+    private static class RateLimitingEndpointHandler {
+
+        private static final TimeValue NO_TASKS_AVAILABLE = TimeValue.MAX_VALUE;
+        private static final TimeValue EXECUTED_A_TASK = TimeValue.ZERO;
+        private static final Logger logger = LogManager.getLogger(RateLimitingEndpointHandler.class);
+        private static final int ACCUMULATED_TOKENS_LIMIT = 1;
+
+        private final AdjustableCapacityBlockingQueue<RejectableTask> queue;
+        private final Supplier<Boolean> isShutdownMethod;
+        private final RequestSender requestSender;
+        private final String id;
+        private final AtomicReference<Instant> timeOfLastEnqueue = new AtomicReference<>();
+        private final Clock clock;
+        private final RateLimiter rateLimiter;
+        private final RequestExecutorServiceSettings requestExecutorServiceSettings;
+
+        RateLimitingEndpointHandler(
+            String id,
+            AdjustableCapacityBlockingQueue.QueueCreator<RejectableTask> createQueue,
+            RequestExecutorServiceSettings settings,
+            RequestSender requestSender,
+            Clock clock,
+            RateLimitSettings rateLimitSettings,
+            Supplier<Boolean> isShutdownMethod,
+            RateLimiterCreator rateLimiterCreator
+        ) {
+            this.requestExecutorServiceSettings = Objects.requireNonNull(settings);
+            this.id = Objects.requireNonNull(id);
+            this.queue = new AdjustableCapacityBlockingQueue<>(createQueue, settings.getQueueCapacity());
+            this.requestSender = Objects.requireNonNull(requestSender);
+            this.clock = Objects.requireNonNull(clock);
+            this.isShutdownMethod = Objects.requireNonNull(isShutdownMethod);
+
+            Objects.requireNonNull(rateLimitSettings);
+            Objects.requireNonNull(rateLimiterCreator);
+            rateLimiter = rateLimiterCreator.create(
+                ACCUMULATED_TOKENS_LIMIT,
+                rateLimitSettings.requestsPerTimeUnit(),
+                rateLimitSettings.timeUnit()
             );
 
-            task.onRejection(rejected);
-            return;
         }
 
-        boolean added = queue.offer(task);
-        if (added == false) {
-            EsRejectedExecutionException rejected = new EsRejectedExecutionException(
-                format("Failed to execute task because the http executor service [%s] queue is full", serviceName),
-                false
-            );
+        public void init() {
+            requestExecutorServiceSettings.registerQueueCapacityCallback(id, this::onCapacityChange);
+        }
 
-            task.onRejection(rejected);
-        } else if (isShutdown()) {
-            // It is possible that a shutdown and notification request occurred after we initially checked for shutdown above
-            // If the task was added after the queue was already drained it could sit there indefinitely. So let's check again if
-            // we shut down and if so we'll redo the notification
-            notifyRequestsOfShutdown();
+        private void onCapacityChange(int capacity) {
+            logger.debug(() -> Strings.format("Executor service grouping [%s] setting queue capacity to [%s]", id, capacity));
+
+            try {
+                queue.setCapacity(capacity);
+            } catch (Exception e) {
+                logger.warn(format("Executor service grouping [%s] failed to set the capacity of the task queue to [%s]", id, capacity), e);
+            }
         }
-    }
 
-    // default for testing
-    int remainingQueueCapacity() {
-        return queue.remainingCapacity();
+        public int queueSize() {
+            return queue.size();
+        }
+
+        public boolean isShutdown() {
+            return isShutdownMethod.get();
+        }
+
+        public Instant timeOfLastEnqueue() {
+            return timeOfLastEnqueue.get();
+        }
+
+        public synchronized TimeValue executeEnqueuedTask() {
+            try {
+                return executeEnqueuedTaskInternal();
+            } catch (Exception e) {
+                logger.warn(format("Executor service grouping [%s] failed to execute request", id), e);
+                // we tried to do some work but failed, so we'll say we did something to try looking for more work
+                return EXECUTED_A_TASK;
+            }
+        }
+
+        private TimeValue executeEnqueuedTaskInternal() {
+            var timeBeforeAvailableToken = rateLimiter.timeToReserve(1);
+            if (shouldExecuteImmediately(timeBeforeAvailableToken) == false) {
+                return timeBeforeAvailableToken;
+            }
+
+            var task = queue.poll();
+
+            // TODO Batching - in a situation where no new tasks are queued we'll want to execute any prepared tasks
+            // So we'll need to check for null and call a helper method executePreparedTasks()
+
+            if (shouldExecuteTask(task) == false) {
+                return NO_TASKS_AVAILABLE;
+            }
+
+            // We should never have to wait because we checked above
+            var reserveRes = rateLimiter.reserve(1);
+            assert shouldExecuteImmediately(reserveRes) : "Reserving request tokens required a sleep when it should not have";
+
+            task.getRequestManager()
+                .execute(task.getQuery(), task.getInput(), requestSender, task.getRequestCompletedFunction(), task.getListener());
+            return EXECUTED_A_TASK;
+        }
+
+        private static boolean shouldExecuteTask(RejectableTask task) {
+            return task != null && isNoopRequest(task) == false && task.hasCompleted() == false;
+        }
+
+        private static boolean isNoopRequest(InferenceRequest inferenceRequest) {
+            return inferenceRequest.getRequestManager() == null
+                || inferenceRequest.getInput() == null
+                || inferenceRequest.getListener() == null;
+        }
+
+        private static boolean shouldExecuteImmediately(TimeValue delay) {
+            return delay.duration() == 0;
+        }
+
+        public void enqueue(RequestTask task) {
+            timeOfLastEnqueue.set(Instant.now(clock));
+
+            if (isShutdown()) {
+                EsRejectedExecutionException rejected = new EsRejectedExecutionException(
+                    format(
+                        "Failed to enqueue task for inference id [%s] because the request service [%s] has already shutdown",
+                        task.getRequestManager().inferenceEntityId(),
+                        id
+                    ),
+                    true
+                );
+
+                task.onRejection(rejected);
+                return;
+            }
+
+            var addedToQueue = queue.offer(task);
+
+            if (addedToQueue == false) {
+                EsRejectedExecutionException rejected = new EsRejectedExecutionException(
+                    format(
+                        "Failed to execute task for inference id [%s] because the request service [%s] queue is full",
+                        task.getRequestManager().inferenceEntityId(),
+                        id
+                    ),
+                    false
+                );
+
+                task.onRejection(rejected);
+            } else if (isShutdown()) {
+                notifyRequestsOfShutdown();
+            }
+        }
+
+        public synchronized void notifyRequestsOfShutdown() {
+            assert isShutdown() : "Requests should only be notified if the executor is shutting down";
+
+            try {
+                List<RejectableTask> notExecuted = new ArrayList<>();
+                queue.drainTo(notExecuted);
+
+                rejectTasks(notExecuted);
+            } catch (Exception e) {
+                logger.warn(format("Failed to notify tasks of executor service grouping [%s] shutdown", id));
+            }
+        }
+
+        private void rejectTasks(List<RejectableTask> tasks) {
+            for (var task : tasks) {
+                rejectTaskForShutdown(task);
+            }
+        }
+
+        private void rejectTaskForShutdown(RejectableTask task) {
+            try {
+                task.onRejection(
+                    new EsRejectedExecutionException(
+                        format(
+                            "Failed to send request, request service [%s] for inference id [%s] has shutdown prior to executing request",
+                            id,
+                            task.getRequestManager().inferenceEntityId()
+                        ),
+                        true
+                    )
+                );
+            } catch (Exception e) {
+                logger.warn(
+                    format(
+                        "Failed to notify request for inference id [%s] of rejection after executor service grouping [%s] shutdown",
+                        task.getRequestManager().inferenceEntityId(),
+                        id
+                    )
+                );
+            }
+        }
+
+        public int remainingCapacity() {
+            return queue.remainingCapacity();
+        }
+
+        public void close() {
+            requestExecutorServiceSettings.deregisterQueueCapacityCallback(id);
+        }
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettings.java
index 86825035f2d05..616ef7a40068b 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettings.java
@@ -10,9 +10,12 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 
-import java.util.ArrayList;
+import java.time.Duration;
 import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.function.Consumer;
 
 public class RequestExecutorServiceSettings {
@@ -29,37 +32,108 @@ public class RequestExecutorServiceSettings {
         Setting.Property.Dynamic
     );
 
+    private static final TimeValue DEFAULT_TASK_POLL_FREQUENCY_TIME = TimeValue.timeValueMillis(50);
+    /**
+     * Defines how often all the rate limit groups are polled for tasks. Setting this to very low number could result
+     * in a busy loop if there are no tasks available to handle.
+     */
+    static final Setting<TimeValue> TASK_POLL_FREQUENCY_SETTING = Setting.timeSetting(
+        "xpack.inference.http.request_executor.task_poll_frequency",
+        DEFAULT_TASK_POLL_FREQUENCY_TIME,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    private static final TimeValue DEFAULT_RATE_LIMIT_GROUP_CLEANUP_INTERVAL = TimeValue.timeValueDays(1);
+    /**
+     * Defines how often a thread will check for rate limit groups that are stale.
+     */
+    static final Setting<TimeValue> RATE_LIMIT_GROUP_CLEANUP_INTERVAL_SETTING = Setting.timeSetting(
+        "xpack.inference.http.request_executor.rate_limit_group_cleanup_interval",
+        DEFAULT_RATE_LIMIT_GROUP_CLEANUP_INTERVAL,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    private static final TimeValue DEFAULT_RATE_LIMIT_GROUP_STALE_DURATION = TimeValue.timeValueDays(10);
+    /**
+     * Defines the amount of time it takes to classify a rate limit group as stale. Once it is classified as stale,
+     * it can be removed when the cleanup thread executes.
+     */
+    static final Setting<TimeValue> RATE_LIMIT_GROUP_STALE_DURATION_SETTING = Setting.timeSetting(
+        "xpack.inference.http.request_executor.rate_limit_group_stale_duration",
+        DEFAULT_RATE_LIMIT_GROUP_STALE_DURATION,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
     public static List<Setting<?>> getSettingsDefinitions() {
-        return List.of(TASK_QUEUE_CAPACITY_SETTING);
+        return List.of(
+            TASK_QUEUE_CAPACITY_SETTING,
+            TASK_POLL_FREQUENCY_SETTING,
+            RATE_LIMIT_GROUP_CLEANUP_INTERVAL_SETTING,
+            RATE_LIMIT_GROUP_STALE_DURATION_SETTING
+        );
     }
 
     private volatile int queueCapacity;
-    private final List<Consumer<Integer>> queueCapacityCallbacks = new ArrayList<Consumer<Integer>>();
+    private volatile TimeValue taskPollFrequency;
+    private volatile Duration rateLimitGroupStaleDuration;
+    private final ConcurrentMap<String, Consumer<Integer>> queueCapacityCallbacks = new ConcurrentHashMap<>();
 
     public RequestExecutorServiceSettings(Settings settings, ClusterService clusterService) {
         queueCapacity = TASK_QUEUE_CAPACITY_SETTING.get(settings);
+        taskPollFrequency = TASK_POLL_FREQUENCY_SETTING.get(settings);
+        setRateLimitGroupStaleDuration(RATE_LIMIT_GROUP_STALE_DURATION_SETTING.get(settings));
 
         addSettingsUpdateConsumers(clusterService);
     }
 
     private void addSettingsUpdateConsumers(ClusterService clusterService) {
         clusterService.getClusterSettings().addSettingsUpdateConsumer(TASK_QUEUE_CAPACITY_SETTING, this::setQueueCapacity);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(TASK_POLL_FREQUENCY_SETTING, this::setTaskPollFrequency);
+        clusterService.getClusterSettings()
+            .addSettingsUpdateConsumer(RATE_LIMIT_GROUP_STALE_DURATION_SETTING, this::setRateLimitGroupStaleDuration);
     }
 
     // default for testing
     void setQueueCapacity(int queueCapacity) {
         this.queueCapacity = queueCapacity;
 
-        for (var callback : queueCapacityCallbacks) {
+        for (var callback : queueCapacityCallbacks.values()) {
             callback.accept(queueCapacity);
         }
     }
 
-    void registerQueueCapacityCallback(Consumer<Integer> onChangeCapacityCallback) {
-        queueCapacityCallbacks.add(onChangeCapacityCallback);
+    private void setTaskPollFrequency(TimeValue taskPollFrequency) {
+        this.taskPollFrequency = taskPollFrequency;
+    }
+
+    private void setRateLimitGroupStaleDuration(TimeValue staleDuration) {
+        rateLimitGroupStaleDuration = toDuration(staleDuration);
+    }
+
+    private static Duration toDuration(TimeValue timeValue) {
+        return Duration.of(timeValue.duration(), timeValue.timeUnit().toChronoUnit());
+    }
+
+    void registerQueueCapacityCallback(String id, Consumer<Integer> onChangeCapacityCallback) {
+        queueCapacityCallbacks.put(id, onChangeCapacityCallback);
+    }
+
+    void deregisterQueueCapacityCallback(String id) {
+        queueCapacityCallbacks.remove(id);
     }
 
     int getQueueCapacity() {
         return queueCapacity;
     }
+
+    TimeValue getTaskPollFrequency() {
+        return taskPollFrequency;
+    }
+
+    Duration getRateLimitGroupStaleDuration() {
+        return rateLimitGroupStaleDuration;
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManager.java
index 7d3cca596f1d0..79ef1b56ad231 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManager.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManager.java
@@ -7,7 +7,6 @@
 
 package org.elasticsearch.xpack.inference.external.http.sender;
 
-import org.apache.http.client.protocol.HttpClientContext;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.InferenceServiceResults;
@@ -21,14 +20,17 @@
  * A contract for constructing a {@link Runnable} to handle sending an inference request to a 3rd party service.
  */
 public interface RequestManager extends RateLimitable {
-    Runnable create(
+    void execute(
         @Nullable String query,
         List<String> input,
         RequestSender requestSender,
         Supplier<Boolean> hasRequestCompletedFunction,
-        HttpClientContext context,
         ActionListener<InferenceServiceResults> listener
     );
 
+    // TODO For batching we'll add 2 new method: prepare(query, input, ...) which will allow the individual
+    // managers to implement their own batching
+    // executePreparedRequest() which will execute all prepared requests aka sends the batch
+
     String inferenceEntityId();
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java
index 738592464232c..7a5f482412289 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java
@@ -111,7 +111,7 @@ public void onRejection(Exception e) {
     }
 
     @Override
-    public RequestManager getRequestCreator() {
+    public RequestManager getRequestManager() {
         return requestCreator;
     }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManager.java
deleted file mode 100644
index 494c77964080f..0000000000000
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManager.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.http.sender;
-
-import org.apache.http.client.protocol.HttpClientContext;
-import org.elasticsearch.xpack.inference.external.http.retry.RetryingHttpSender;
-
-import java.util.Objects;
-
-/**
- * Handles executing a single inference request at a time.
- */
-public class SingleRequestManager {
-
-    protected RetryingHttpSender requestSender;
-
-    public SingleRequestManager(RetryingHttpSender requestSender) {
-        this.requestSender = Objects.requireNonNull(requestSender);
-    }
-
-    public void execute(InferenceRequest inferenceRequest, HttpClientContext context) {
-        if (isNoopRequest(inferenceRequest) || inferenceRequest.hasCompleted()) {
-            return;
-        }
-
-        inferenceRequest.getRequestCreator()
-            .create(
-                inferenceRequest.getQuery(),
-                inferenceRequest.getInput(),
-                requestSender,
-                inferenceRequest.getRequestCompletedFunction(),
-                context,
-                inferenceRequest.getListener()
-            )
-            .run();
-    }
-
-    private static boolean isNoopRequest(InferenceRequest inferenceRequest) {
-        return inferenceRequest.getRequestCreator() == null
-            || inferenceRequest.getInput() == null
-            || inferenceRequest.getListener() == null;
-    }
-}
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java
index 24c0ab2cd893e..1c64f505402d8 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java
@@ -31,7 +31,7 @@ public abstract class SenderService implements InferenceService {
 
     public SenderService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
         Objects.requireNonNull(factory);
-        sender = factory.createSender(name());
+        sender = factory.createSender();
         this.serviceComponents = Objects.requireNonNull(serviceComponents);
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java
index c488eac422401..f30773962854a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java
@@ -56,7 +56,7 @@
 
 public class AzureAiStudioService extends SenderService {
 
-    private static final String NAME = "azureaistudio";
+    static final String NAME = "azureaistudio";
 
     public AzureAiStudioService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
         super(factory, serviceComponents);
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceSettings.java
index 10c57e19b6403..03034ae70c2b6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceSettings.java
@@ -44,7 +44,13 @@ protected static BaseAzureAiStudioCommonFields fromMap(
         ConfigurationParseContext context
     ) {
         String target = extractRequiredString(map, TARGET_FIELD, ModelConfigurations.SERVICE_SETTINGS, validationException);
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            AzureAiStudioService.NAME,
+            context
+        );
         AzureAiStudioEndpointType endpointType = extractRequiredEnum(
             map,
             ENDPOINT_TYPE_FIELD,
@@ -118,13 +124,13 @@ public void writeTo(StreamOutput out) throws IOException {
 
     protected void addXContentFields(XContentBuilder builder, Params params) throws IOException {
         this.addExposedXContentFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
     }
 
     protected void addExposedXContentFields(XContentBuilder builder, Params params) throws IOException {
         builder.field(TARGET_FIELD, this.target);
         builder.field(PROVIDER_FIELD, this.provider);
         builder.field(ENDPOINT_TYPE_FIELD, this.endpointType);
+        rateLimitSettings.toXContent(builder, params);
     }
 
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java
index e0e48ab20a86b..26bf6f1648d97 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java
@@ -135,7 +135,15 @@ private static AzureOpenAiModel createModel(
                 );
             }
             case COMPLETION -> {
-                return new AzureOpenAiCompletionModel(inferenceEntityId, taskType, NAME, serviceSettings, taskSettings, secretSettings);
+                return new AzureOpenAiCompletionModel(
+                    inferenceEntityId,
+                    taskType,
+                    NAME,
+                    serviceSettings,
+                    taskSettings,
+                    secretSettings,
+                    context
+                );
             }
             default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
         }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java
index 05cb663453542..c4146b2ba2d30 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionModel.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.azureopenai.AzureOpenAiActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.azureopenai.AzureOpenAiUtils;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiModel;
 import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiSecretSettings;
 
@@ -37,13 +38,14 @@ public AzureOpenAiCompletionModel(
         String service,
         Map<String, Object> serviceSettings,
         Map<String, Object> taskSettings,
-        @Nullable Map<String, Object> secrets
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
     ) {
         this(
             inferenceEntityId,
             taskType,
             service,
-            AzureOpenAiCompletionServiceSettings.fromMap(serviceSettings),
+            AzureOpenAiCompletionServiceSettings.fromMap(serviceSettings, context),
             AzureOpenAiCompletionTaskSettings.fromMap(taskSettings),
             AzureOpenAiSecretSettings.fromMap(secrets)
         );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java
index ba503b2bbdc4b..92dc461d9008c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettings.java
@@ -17,7 +17,9 @@
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiService;
 import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -55,10 +57,10 @@ public class AzureOpenAiCompletionServiceSettings extends FilteredXContentObject
      */
     private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(120);
 
-    public static AzureOpenAiCompletionServiceSettings fromMap(Map<String, Object> map) {
+    public static AzureOpenAiCompletionServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
         ValidationException validationException = new ValidationException();
 
-        var settings = fromMap(map, validationException);
+        var settings = fromMap(map, validationException, context);
 
         if (validationException.validationErrors().isEmpty() == false) {
             throw validationException;
@@ -69,12 +71,19 @@ public static AzureOpenAiCompletionServiceSettings fromMap(Map<String, Object> m
 
     private static AzureOpenAiCompletionServiceSettings.CommonFields fromMap(
         Map<String, Object> map,
-        ValidationException validationException
+        ValidationException validationException,
+        ConfigurationParseContext context
     ) {
         String resourceName = extractRequiredString(map, RESOURCE_NAME, ModelConfigurations.SERVICE_SETTINGS, validationException);
         String deploymentId = extractRequiredString(map, DEPLOYMENT_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
         String apiVersion = extractRequiredString(map, API_VERSION, ModelConfigurations.SERVICE_SETTINGS, validationException);
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            AzureOpenAiService.NAME,
+            context
+        );
 
         return new AzureOpenAiCompletionServiceSettings.CommonFields(resourceName, deploymentId, apiVersion, rateLimitSettings);
     }
@@ -137,7 +146,6 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
         builder.startObject();
 
         toXContentFragmentOfExposedFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
 
         builder.endObject();
         return builder;
@@ -148,6 +156,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
         builder.field(RESOURCE_NAME, resourceName);
         builder.field(DEPLOYMENT_ID, deploymentId);
         builder.field(API_VERSION, apiVersion);
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java
index 33bb0fdb07c58..1c426815a83c0 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.ServiceUtils;
 import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiService;
 import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -90,7 +91,13 @@ private static CommonFields fromMap(
         Integer dims = removeAsType(map, DIMENSIONS, Integer.class);
         Integer maxTokens = removeAsType(map, MAX_INPUT_TOKENS, Integer.class);
         SimilarityMeasure similarity = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException);
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            AzureOpenAiService.NAME,
+            context
+        );
 
         Boolean dimensionsSetByUser = extractOptionalBoolean(map, DIMENSIONS_SET_BY_USER, validationException);
 
@@ -245,8 +252,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
 
         toXContentFragmentOfExposedFields(builder, params);
-
-        rateLimitSettings.toXContent(builder, params);
         builder.field(DIMENSIONS_SET_BY_USER, dimensionsSetByUser);
 
         builder.endObject();
@@ -268,6 +273,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
         if (similarity != null) {
             builder.field(SIMILARITY, similarity);
         }
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java
index 11dbf673ab7bd..4c673026d7efb 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java
@@ -51,6 +51,11 @@
 public class CohereService extends SenderService {
     public static final String NAME = "cohere";
 
+    // TODO Batching - We'll instantiate a batching class within the services that want to support it and pass it through to
+    // the Cohere*RequestManager via the CohereActionCreator class
+    // The reason it needs to be done here is that the batching logic needs to hold state but the *RequestManagers are instantiated
+    // on every request
+
     public CohereService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) {
         super(factory, serviceComponents);
     }
@@ -131,7 +136,15 @@ private static CohereModel createModel(
                 context
             );
             case RERANK -> new CohereRerankModel(inferenceEntityId, taskType, NAME, serviceSettings, taskSettings, secretSettings, context);
-            case COMPLETION -> new CohereCompletionModel(inferenceEntityId, taskType, NAME, serviceSettings, taskSettings, secretSettings);
+            case COMPLETION -> new CohereCompletionModel(
+                inferenceEntityId,
+                taskType,
+                NAME,
+                serviceSettings,
+                taskSettings,
+                secretSettings,
+                context
+            );
             default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
         };
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java
index b23f6f188d8c5..d477a8c5a5f55 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java
@@ -58,7 +58,13 @@ public static CohereServiceSettings fromMap(Map<String, Object> map, Configurati
         Integer maxInputTokens = removeAsType(map, MAX_INPUT_TOKENS, Integer.class);
         URI uri = convertToUri(url, URL, ModelConfigurations.SERVICE_SETTINGS, validationException);
         String oldModelId = extractOptionalString(map, OLD_MODEL_ID_FIELD, ModelConfigurations.SERVICE_SETTINGS, validationException);
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            CohereService.NAME,
+            context
+        );
 
         String modelId = extractOptionalString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
 
@@ -173,10 +179,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     }
 
     public XContentBuilder toXContentFragment(XContentBuilder builder, Params params) throws IOException {
-        toXContentFragmentOfExposedFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
-
-        return builder;
+        return toXContentFragmentOfExposedFields(builder, params);
     }
 
     @Override
@@ -196,6 +199,7 @@ public XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder
         if (modelId != null) {
             builder.field(MODEL_ID, modelId);
         }
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModel.java
index 761081d4d723c..bec4f5a0b5c85 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModel.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.cohere.CohereActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.cohere.CohereModel;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
@@ -30,13 +31,14 @@ public CohereCompletionModel(
         String service,
         Map<String, Object> serviceSettings,
         Map<String, Object> taskSettings,
-        @Nullable Map<String, Object> secrets
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
     ) {
         this(
             modelId,
             taskType,
             service,
-            CohereCompletionServiceSettings.fromMap(serviceSettings),
+            CohereCompletionServiceSettings.fromMap(serviceSettings, context),
             EmptyTaskSettings.INSTANCE,
             DefaultSecretSettings.fromMap(secrets)
         );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java
index 2a22f6333f1a2..ba9e81b461f9f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java
@@ -16,7 +16,9 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.cohere.CohereRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.cohere.CohereService;
 import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -39,12 +41,18 @@ public class CohereCompletionServiceSettings extends FilteredXContentObject impl
     // 10K requests per minute
     private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(10_000);
 
-    public static CohereCompletionServiceSettings fromMap(Map<String, Object> map) {
+    public static CohereCompletionServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
         ValidationException validationException = new ValidationException();
 
         String url = extractOptionalString(map, URL, ModelConfigurations.SERVICE_SETTINGS, validationException);
         URI uri = convertToUri(url, URL, ModelConfigurations.SERVICE_SETTINGS, validationException);
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            CohereService.NAME,
+            context
+        );
         String modelId = extractOptionalString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
 
         if (validationException.validationErrors().isEmpty() == false) {
@@ -94,7 +102,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
 
         toXContentFragmentOfExposedFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
 
         builder.endObject();
         return builder;
@@ -127,6 +134,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
         if (modelId != null) {
             builder.field(MODEL_ID, modelId);
         }
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
index f8720448b0f4f..cfa8566495143 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java
@@ -108,7 +108,8 @@ private static GoogleAiStudioModel createModel(
                 NAME,
                 serviceSettings,
                 taskSettings,
-                secretSettings
+                secretSettings,
+                context
             );
             case TEXT_EMBEDDING -> new GoogleAiStudioEmbeddingsModel(
                 inferenceEntityId,
@@ -116,7 +117,8 @@ private static GoogleAiStudioModel createModel(
                 NAME,
                 serviceSettings,
                 taskSettings,
-                secretSettings
+                secretSettings,
+                context
             );
             default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
         };
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java
index eafb0c372202c..8fa2ac0148716 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModel.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioUtils;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioModel;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
@@ -37,13 +38,14 @@ public GoogleAiStudioCompletionModel(
         String service,
         Map<String, Object> serviceSettings,
         Map<String, Object> taskSettings,
-        Map<String, Object> secrets
+        Map<String, Object> secrets,
+        ConfigurationParseContext context
     ) {
         this(
             inferenceEntityId,
             taskType,
             service,
-            GoogleAiStudioCompletionServiceSettings.fromMap(serviceSettings),
+            GoogleAiStudioCompletionServiceSettings.fromMap(serviceSettings, context),
             EmptyTaskSettings.INSTANCE,
             DefaultSecretSettings.fromMap(secrets)
         );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettings.java
index f8f343be8eb4c..7c0b812ee213b 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettings.java
@@ -16,7 +16,9 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioService;
 import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -40,11 +42,17 @@ public class GoogleAiStudioCompletionServiceSettings extends FilteredXContentObj
      */
     private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(360);
 
-    public static GoogleAiStudioCompletionServiceSettings fromMap(Map<String, Object> map) {
+    public static GoogleAiStudioCompletionServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
         ValidationException validationException = new ValidationException();
 
         String model = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            GoogleAiStudioService.NAME,
+            context
+        );
 
         if (validationException.validationErrors().isEmpty() == false) {
             throw validationException;
@@ -82,7 +90,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
 
         toXContentFragmentOfExposedFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
 
         builder.endObject();
         return builder;
@@ -107,6 +114,7 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException {
         builder.field(MODEL_ID, modelId);
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
index ad106797de51b..af19e26f3e97a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsModel.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.googleaistudio.GoogleAiStudioActionVisitor;
 import org.elasticsearch.xpack.inference.external.request.googleaistudio.GoogleAiStudioUtils;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioModel;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
@@ -37,13 +38,14 @@ public GoogleAiStudioEmbeddingsModel(
         String service,
         Map<String, Object> serviceSettings,
         Map<String, Object> taskSettings,
-        Map<String, Object> secrets
+        Map<String, Object> secrets,
+        ConfigurationParseContext context
     ) {
         this(
             inferenceEntityId,
             taskType,
             service,
-            GoogleAiStudioEmbeddingsServiceSettings.fromMap(serviceSettings),
+            GoogleAiStudioEmbeddingsServiceSettings.fromMap(serviceSettings, context),
             EmptyTaskSettings.INSTANCE,
             DefaultSecretSettings.fromMap(secrets)
         );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java
index 07d07dc533f06..7608f48d0638d 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettings.java
@@ -18,7 +18,9 @@
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.inference.SimilarityMeasure;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioService;
 import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -47,7 +49,7 @@ public class GoogleAiStudioEmbeddingsServiceSettings extends FilteredXContentObj
      */
     private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(360);
 
-    public static GoogleAiStudioEmbeddingsServiceSettings fromMap(Map<String, Object> map) {
+    public static GoogleAiStudioEmbeddingsServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
         ValidationException validationException = new ValidationException();
 
         String model = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
@@ -59,7 +61,13 @@ public static GoogleAiStudioEmbeddingsServiceSettings fromMap(Map<String, Object
         );
         SimilarityMeasure similarityMeasure = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException);
         Integer dims = extractOptionalPositiveInteger(map, DIMENSIONS, ModelConfigurations.SERVICE_SETTINGS, validationException);
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            GoogleAiStudioService.NAME,
+            context
+        );
 
         if (validationException.validationErrors().isEmpty() == false) {
             throw validationException;
@@ -134,7 +142,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
 
         toXContentFragmentOfExposedFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
 
         builder.endObject();
         return builder;
@@ -174,6 +181,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
         if (similarity != null) {
             builder.field(SIMILARITY, similarity);
         }
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java
index ebb6c10207f4e..ef034816f762c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseService.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.xpack.inference.external.action.huggingface.HuggingFaceActionCreator;
 import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.SenderService;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
 
@@ -62,7 +63,8 @@ public void parseRequestConfig(
                 taskType,
                 serviceSettingsMap,
                 serviceSettingsMap,
-                TaskType.unsupportedTaskTypeErrorMsg(taskType, name())
+                TaskType.unsupportedTaskTypeErrorMsg(taskType, name()),
+                ConfigurationParseContext.REQUEST
             );
 
             throwIfNotEmptyMap(config, name());
@@ -89,7 +91,8 @@ public HuggingFaceModel parsePersistedConfigWithSecrets(
             taskType,
             serviceSettingsMap,
             secretSettingsMap,
-            parsePersistedConfigErrorMsg(inferenceEntityId, name())
+            parsePersistedConfigErrorMsg(inferenceEntityId, name()),
+            ConfigurationParseContext.PERSISTENT
         );
     }
 
@@ -97,7 +100,14 @@ public HuggingFaceModel parsePersistedConfigWithSecrets(
     public HuggingFaceModel parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map<String, Object> config) {
         Map<String, Object> serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS);
 
-        return createModel(inferenceEntityId, taskType, serviceSettingsMap, null, parsePersistedConfigErrorMsg(inferenceEntityId, name()));
+        return createModel(
+            inferenceEntityId,
+            taskType,
+            serviceSettingsMap,
+            null,
+            parsePersistedConfigErrorMsg(inferenceEntityId, name()),
+            ConfigurationParseContext.PERSISTENT
+        );
     }
 
     protected abstract HuggingFaceModel createModel(
@@ -105,7 +115,8 @@ protected abstract HuggingFaceModel createModel(
         TaskType taskType,
         Map<String, Object> serviceSettings,
         Map<String, Object> secretSettings,
-        String failureMessage
+        String failureMessage,
+        ConfigurationParseContext context
     );
 
     @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java
index d8c383d2b4a67..c0438b3759a65 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
 import org.elasticsearch.xpack.inference.services.ServiceUtils;
 import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserModel;
@@ -36,11 +37,19 @@ protected HuggingFaceModel createModel(
         TaskType taskType,
         Map<String, Object> serviceSettings,
         @Nullable Map<String, Object> secretSettings,
-        String failureMessage
+        String failureMessage,
+        ConfigurationParseContext context
     ) {
         return switch (taskType) {
-            case TEXT_EMBEDDING -> new HuggingFaceEmbeddingsModel(inferenceEntityId, taskType, NAME, serviceSettings, secretSettings);
-            case SPARSE_EMBEDDING -> new HuggingFaceElserModel(inferenceEntityId, taskType, NAME, serviceSettings, secretSettings);
+            case TEXT_EMBEDDING -> new HuggingFaceEmbeddingsModel(
+                inferenceEntityId,
+                taskType,
+                NAME,
+                serviceSettings,
+                secretSettings,
+                context
+            );
+            case SPARSE_EMBEDDING -> new HuggingFaceElserModel(inferenceEntityId, taskType, NAME, serviceSettings, secretSettings, context);
             default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
         };
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java
index af2c433663ac4..fc31b1e518dd9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.inference.SimilarityMeasure;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -43,14 +44,20 @@ public class HuggingFaceServiceSettings extends FilteredXContentObject implement
     // 3000 requests per minute
     private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(3000);
 
-    public static HuggingFaceServiceSettings fromMap(Map<String, Object> map) {
+    public static HuggingFaceServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
         ValidationException validationException = new ValidationException();
         var uri = extractUri(map, URL, validationException);
 
         SimilarityMeasure similarityMeasure = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException);
         Integer dims = removeAsType(map, DIMENSIONS, Integer.class);
         Integer maxInputTokens = removeAsType(map, MAX_INPUT_TOKENS, Integer.class);
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            HuggingFaceService.NAME,
+            context
+        );
 
         if (validationException.validationErrors().isEmpty() == false) {
             throw validationException;
@@ -119,7 +126,6 @@ public HuggingFaceServiceSettings(StreamInput in) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         toXContentFragmentOfExposedFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
         builder.endObject();
         return builder;
     }
@@ -136,6 +142,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
         if (maxInputTokens != null) {
             builder.field(MAX_INPUT_TOKENS, maxInputTokens);
         }
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java
index 9010571ea2e55..8132089d8dc99 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.huggingface.HuggingFaceActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceModel;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
@@ -24,13 +25,14 @@ public HuggingFaceElserModel(
         TaskType taskType,
         String service,
         Map<String, Object> serviceSettings,
-        @Nullable Map<String, Object> secrets
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
     ) {
         this(
             inferenceEntityId,
             taskType,
             service,
-            HuggingFaceElserServiceSettings.fromMap(serviceSettings),
+            HuggingFaceElserServiceSettings.fromMap(serviceSettings, context),
             DefaultSecretSettings.fromMap(secrets)
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java
index 2587b2737e164..d3099e96ee7c1 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceBaseService;
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceModel;
@@ -38,10 +39,11 @@ protected HuggingFaceModel createModel(
         TaskType taskType,
         Map<String, Object> serviceSettings,
         @Nullable Map<String, Object> secretSettings,
-        String failureMessage
+        String failureMessage,
+        ConfigurationParseContext context
     ) {
         return switch (taskType) {
-            case SPARSE_EMBEDDING -> new HuggingFaceElserModel(inferenceEntityId, taskType, NAME, serviceSettings, secretSettings);
+            case SPARSE_EMBEDDING -> new HuggingFaceElserModel(inferenceEntityId, taskType, NAME, serviceSettings, secretSettings, context);
             default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
         };
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java
index 1f337de450ef9..8b4bd61649de0 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java
@@ -15,7 +15,9 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceService;
 import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -40,10 +42,16 @@ public class HuggingFaceElserServiceSettings extends FilteredXContentObject
     // 3000 requests per minute
     private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(3000);
 
-    public static HuggingFaceElserServiceSettings fromMap(Map<String, Object> map) {
+    public static HuggingFaceElserServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
         ValidationException validationException = new ValidationException();
         var uri = extractUri(map, URL, validationException);
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            HuggingFaceService.NAME,
+            context
+        );
 
         if (validationException.validationErrors().isEmpty() == false) {
             throw validationException;
@@ -93,7 +101,6 @@ public int maxInputTokens() {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         toXContentFragmentOfExposedFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
         builder.endObject();
 
         return builder;
@@ -103,6 +110,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException {
         builder.field(URL, uri.toString());
         builder.field(MAX_INPUT_TOKENS, ELSER_TOKEN_LIMIT);
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/embeddings/HuggingFaceEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/embeddings/HuggingFaceEmbeddingsModel.java
index 1cee26558b490..fedd6380d035f 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/embeddings/HuggingFaceEmbeddingsModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/embeddings/HuggingFaceEmbeddingsModel.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.huggingface.HuggingFaceActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceModel;
 import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceServiceSettings;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
@@ -25,13 +26,14 @@ public HuggingFaceEmbeddingsModel(
         TaskType taskType,
         String service,
         Map<String, Object> serviceSettings,
-        @Nullable Map<String, Object> secrets
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
     ) {
         this(
             inferenceEntityId,
             taskType,
             service,
-            HuggingFaceServiceSettings.fromMap(serviceSettings),
+            HuggingFaceServiceSettings.fromMap(serviceSettings, context),
             DefaultSecretSettings.fromMap(secrets)
         );
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java
index d2ea8ccbd18bd..62d06a4e0029c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettings.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.inference.SimilarityMeasure;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
+import org.elasticsearch.xpack.inference.services.mistral.MistralService;
 import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -59,7 +60,13 @@ public static MistralEmbeddingsServiceSettings fromMap(Map<String, Object> map,
             ModelConfigurations.SERVICE_SETTINGS,
             validationException
         );
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            MistralService.NAME,
+            context
+        );
         Integer dims = removeAsType(map, DIMENSIONS, Integer.class);
 
         if (validationException.validationErrors().isEmpty() == false) {
@@ -141,7 +148,6 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         this.toXContentFragmentOfExposedFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
         builder.endObject();
         return builder;
     }
@@ -159,6 +165,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
         if (this.maxInputTokens != null) {
             builder.field(MAX_INPUT_TOKENS, this.maxInputTokens);
         }
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java
index 84dfac8903678..04b6ae94d6b53 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java
@@ -138,7 +138,8 @@ private static OpenAiModel createModel(
                 NAME,
                 serviceSettings,
                 taskSettings,
-                secretSettings
+                secretSettings,
+                context
             );
             default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST);
         };
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java
index b1b670c0911f5..7ca93684bc680 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.xpack.inference.external.action.ExecutableAction;
 import org.elasticsearch.xpack.inference.external.action.openai.OpenAiActionVisitor;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiModel;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
@@ -35,13 +36,14 @@ public OpenAiChatCompletionModel(
         String service,
         Map<String, Object> serviceSettings,
         Map<String, Object> taskSettings,
-        @Nullable Map<String, Object> secrets
+        @Nullable Map<String, Object> secrets,
+        ConfigurationParseContext context
     ) {
         this(
             inferenceEntityId,
             taskType,
             service,
-            OpenAiChatCompletionServiceSettings.fromMap(serviceSettings),
+            OpenAiChatCompletionServiceSettings.fromMap(serviceSettings, context),
             OpenAiChatCompletionTaskSettings.fromMap(taskSettings),
             DefaultSecretSettings.fromMap(secrets)
         );
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java
index 5105bb59e048f..04f77da1b1463 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java
@@ -16,7 +16,9 @@
 import org.elasticsearch.inference.ModelConfigurations;
 import org.elasticsearch.inference.ServiceSettings;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.openai.OpenAiService;
 import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -47,7 +49,7 @@ public class OpenAiChatCompletionServiceSettings extends FilteredXContentObject
     // 500 requests per minute
     private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(500);
 
-    public static OpenAiChatCompletionServiceSettings fromMap(Map<String, Object> map) {
+    public static OpenAiChatCompletionServiceSettings fromMap(Map<String, Object> map, ConfigurationParseContext context) {
         ValidationException validationException = new ValidationException();
 
         String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
@@ -58,7 +60,13 @@ public static OpenAiChatCompletionServiceSettings fromMap(Map<String, Object> ma
 
         Integer maxInputTokens = removeAsType(map, MAX_INPUT_TOKENS, Integer.class);
 
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            OpenAiService.NAME,
+            context
+        );
 
         if (validationException.validationErrors().isEmpty() == false) {
             throw validationException;
@@ -142,7 +150,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
 
         toXContentFragmentOfExposedFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
 
         builder.endObject();
         return builder;
@@ -163,6 +170,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
         if (maxInputTokens != null) {
             builder.field(MAX_INPUT_TOKENS, maxInputTokens);
         }
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
index fc479009d3334..080251bf1ba3a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiRateLimitServiceSettings;
+import org.elasticsearch.xpack.inference.services.openai.OpenAiService;
 import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -66,7 +67,7 @@ private static OpenAiEmbeddingsServiceSettings fromPersistentMap(Map<String, Obj
         // passed at that time and never throw.
         ValidationException validationException = new ValidationException();
 
-        var commonFields = fromMap(map, validationException);
+        var commonFields = fromMap(map, validationException, ConfigurationParseContext.PERSISTENT);
 
         Boolean dimensionsSetByUser = removeAsType(map, DIMENSIONS_SET_BY_USER, Boolean.class);
         if (dimensionsSetByUser == null) {
@@ -80,7 +81,7 @@ private static OpenAiEmbeddingsServiceSettings fromPersistentMap(Map<String, Obj
     private static OpenAiEmbeddingsServiceSettings fromRequestMap(Map<String, Object> map) {
         ValidationException validationException = new ValidationException();
 
-        var commonFields = fromMap(map, validationException);
+        var commonFields = fromMap(map, validationException, ConfigurationParseContext.REQUEST);
 
         if (validationException.validationErrors().isEmpty() == false) {
             throw validationException;
@@ -89,7 +90,11 @@ private static OpenAiEmbeddingsServiceSettings fromRequestMap(Map<String, Object
         return new OpenAiEmbeddingsServiceSettings(commonFields, commonFields.dimensions != null);
     }
 
-    private static CommonFields fromMap(Map<String, Object> map, ValidationException validationException) {
+    private static CommonFields fromMap(
+        Map<String, Object> map,
+        ValidationException validationException,
+        ConfigurationParseContext context
+    ) {
 
         String url = extractOptionalString(map, URL, ModelConfigurations.SERVICE_SETTINGS, validationException);
         String organizationId = extractOptionalString(map, ORGANIZATION, ModelConfigurations.SERVICE_SETTINGS, validationException);
@@ -98,7 +103,13 @@ private static CommonFields fromMap(Map<String, Object> map, ValidationException
         Integer dims = removeAsType(map, DIMENSIONS, Integer.class);
         URI uri = convertToUri(url, URL, ModelConfigurations.SERVICE_SETTINGS, validationException);
         String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException);
-        RateLimitSettings rateLimitSettings = RateLimitSettings.of(map, DEFAULT_RATE_LIMIT_SETTINGS, validationException);
+        RateLimitSettings rateLimitSettings = RateLimitSettings.of(
+            map,
+            DEFAULT_RATE_LIMIT_SETTINGS,
+            validationException,
+            OpenAiService.NAME,
+            context
+        );
 
         return new CommonFields(modelId, uri, organizationId, similarity, maxInputTokens, dims, rateLimitSettings);
     }
@@ -258,7 +269,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
 
         toXContentFragmentOfExposedFields(builder, params);
-        rateLimitSettings.toXContent(builder, params);
 
         if (dimensionsSetByUser != null) {
             builder.field(DIMENSIONS_SET_BY_USER, dimensionsSetByUser);
@@ -286,6 +296,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil
         if (maxInputTokens != null) {
             builder.field(MAX_INPUT_TOKENS, maxInputTokens);
         }
+        rateLimitSettings.toXContent(builder, params);
 
         return builder;
     }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java
index cfc375a525dd6..f593ca4e0c603 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettings.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 
 import java.io.IOException;
 import java.util.Map;
@@ -21,19 +22,29 @@
 
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveLong;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty;
+import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
 
 public class RateLimitSettings implements Writeable, ToXContentFragment {
-
     public static final String FIELD_NAME = "rate_limit";
     public static final String REQUESTS_PER_MINUTE_FIELD = "requests_per_minute";
 
     private final long requestsPerTimeUnit;
     private final TimeUnit timeUnit;
 
-    public static RateLimitSettings of(Map<String, Object> map, RateLimitSettings defaultValue, ValidationException validationException) {
+    public static RateLimitSettings of(
+        Map<String, Object> map,
+        RateLimitSettings defaultValue,
+        ValidationException validationException,
+        String serviceName,
+        ConfigurationParseContext context
+    ) {
         Map<String, Object> settings = removeFromMapOrDefaultEmpty(map, FIELD_NAME);
         var requestsPerMinute = extractOptionalPositiveLong(settings, REQUESTS_PER_MINUTE_FIELD, FIELD_NAME, validationException);
 
+        if (ConfigurationParseContext.isRequestContext(context)) {
+            throwIfNotEmptyMap(settings, serviceName);
+        }
+
         return requestsPerMinute == null ? defaultValue : new RateLimitSettings(requestsPerMinute);
     }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionAndCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionAndCreatorTests.java
index 88d408d309a7b..8792234102a94 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionAndCreatorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureaistudio/AzureAiStudioActionAndCreatorTests.java
@@ -44,6 +44,7 @@
 import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
 import static org.elasticsearch.xpack.inference.external.http.retry.RetrySettingsTests.buildSettingsWithRetryFields;
+import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender;
 import static org.elasticsearch.xpack.inference.external.request.azureaistudio.AzureAiStudioRequestFields.API_KEY_HEADER;
 import static org.elasticsearch.xpack.inference.results.ChatCompletionResultsTests.buildExpectationCompletion;
 import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat;
@@ -92,7 +93,7 @@ public void testEmbeddingsRequestAction() throws IOException {
             TruncatorTests.createTruncator()
         );
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             webServer.enqueue(new MockResponse().setResponseCode(200).setBody(testEmbeddingsTokenResponseJson));
@@ -141,7 +142,7 @@ public void testChatCompletionRequestAction() throws IOException {
             TruncatorTests.createTruncator()
         );
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             webServer.enqueue(new MockResponse().setResponseCode(200).setBody(testCompletionTokenResponseJson));
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java
index 0a2a00143b205..72124a6221254 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiActionCreatorTests.java
@@ -44,6 +44,7 @@
 import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
 import static org.elasticsearch.xpack.inference.external.http.retry.RetrySettingsTests.buildSettingsWithRetryFields;
+import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender;
 import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat;
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
 import static org.elasticsearch.xpack.inference.services.azureopenai.completion.AzureOpenAiCompletionModelTests.createCompletionModel;
@@ -82,7 +83,7 @@ public void shutdown() throws IOException {
     public void testCreate_AzureOpenAiEmbeddingsModel() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -132,7 +133,7 @@ public void testCreate_AzureOpenAiEmbeddingsModel() throws IOException {
     public void testCreate_AzureOpenAiEmbeddingsModel_WithoutUser() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -183,7 +184,7 @@ public void testCreate_AzureOpenAiEmbeddingsModel_FailsFromInvalidResponseFormat
         // timeout as zero for no retries
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager, ZERO_TIMEOUT_SETTINGS);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -237,7 +238,7 @@ public void testCreate_AzureOpenAiEmbeddingsModel_FailsFromInvalidResponseFormat
     public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From413StatusCode() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             // note - there is no complete documentation on Azure's error messages
@@ -313,7 +314,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From413StatusC
     public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From400StatusCode() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             // note - there is no complete documentation on Azure's error messages
@@ -389,7 +390,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From400StatusC
     public void testExecute_TruncatesInputBeforeSending() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -440,7 +441,7 @@ public void testExecute_TruncatesInputBeforeSending() throws IOException {
     public void testInfer_AzureOpenAiCompletion_WithOverriddenUser() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -498,7 +499,7 @@ public void testInfer_AzureOpenAiCompletion_WithOverriddenUser() throws IOExcept
     public void testInfer_AzureOpenAiCompletionModel_WithoutUser() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -554,7 +555,7 @@ public void testInfer_AzureOpenAiCompletionModel_FailsFromInvalidResponseFormat(
         // timeout as zero for no retries
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager, ZERO_TIMEOUT_SETTINGS);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             // "choices" missing
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java
index 96127841c17a8..7d52616402405 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiCompletionActionTests.java
@@ -44,6 +44,7 @@
 import static org.elasticsearch.xpack.inference.external.action.azureopenai.AzureOpenAiActionCreatorTests.getContentOfMessageInRequestMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
+import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender;
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
 import static org.elasticsearch.xpack.inference.services.azureopenai.completion.AzureOpenAiCompletionModelTests.createCompletionModel;
 import static org.hamcrest.Matchers.hasSize;
@@ -77,7 +78,7 @@ public void shutdown() throws IOException {
     public void testExecute_ReturnsSuccessfulResponse() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java
index 89cc847321796..4cc7b7c0d9cfc 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsActionTests.java
@@ -43,6 +43,7 @@
 import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty;
 import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
+import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender;
 import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat;
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
 import static org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsModelTests.createModel;
@@ -81,7 +82,7 @@ public void testExecute_ReturnsSuccessfulResponse() throws IOException {
             mockClusterServiceEmpty()
         );
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreatorTests.java
index 9b0371ad51f8c..9ec34e7d8e5c5 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreatorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreatorTests.java
@@ -42,6 +42,7 @@
 import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty;
 import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
+import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender;
 import static org.elasticsearch.xpack.inference.results.ChatCompletionResultsTests.buildExpectationCompletion;
 import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat;
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
@@ -73,7 +74,7 @@ public void shutdown() throws IOException {
     public void testCreate_CohereEmbeddingsModel() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -154,7 +155,7 @@ public void testCreate_CohereEmbeddingsModel() throws IOException {
     public void testCreate_CohereCompletionModel_WithModelSpecified() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -214,7 +215,7 @@ public void testCreate_CohereCompletionModel_WithModelSpecified() throws IOExcep
     public void testCreate_CohereCompletionModel_WithoutModelSpecified() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java
index 12c3d132d1244..0a604980f6c83 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereCompletionActionTests.java
@@ -77,7 +77,7 @@ public void shutdown() throws IOException {
     public void testExecute_ReturnsSuccessfulResponse_WithModelSpecified() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = HttpRequestSenderTests.createSenderWithSingleRequestManager(senderFactory, "test_service")) {
+        try (var sender = HttpRequestSenderTests.createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -138,7 +138,7 @@ public void testExecute_ReturnsSuccessfulResponse_WithModelSpecified() throws IO
     public void testExecute_ReturnsSuccessfulResponse_WithoutModelSpecified() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = HttpRequestSenderTests.createSenderWithSingleRequestManager(senderFactory, "test_service")) {
+        try (var sender = HttpRequestSenderTests.createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -290,7 +290,7 @@ public void testExecute_ThrowsExceptionWithNullUrl() {
     public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = HttpRequestSenderTests.createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java
index dbc97fa2e13d8..9cf6de27b93bc 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java
@@ -81,7 +81,7 @@ public void shutdown() throws IOException {
     public void testExecute_ReturnsSuccessfulResponse() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = HttpRequestSenderTests.createSenderWithSingleRequestManager(senderFactory, "test_service")) {
+        try (var sender = HttpRequestSenderTests.createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -162,7 +162,7 @@ public void testExecute_ReturnsSuccessfulResponse() throws IOException {
     public void testExecute_ReturnsSuccessfulResponse_ForInt8ResponseType() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = HttpRequestSenderTests.createSenderWithSingleRequestManager(senderFactory, "test_service")) {
+        try (var sender = HttpRequestSenderTests.createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java
index 09ef5351eb1fc..9dd465e0276f4 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioCompletionActionTests.java
@@ -74,7 +74,7 @@ public void shutdown() throws IOException {
     public void testExecute_ReturnsSuccessfulResponse() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = HttpRequestSenderTests.createSenderWithSingleRequestManager(senderFactory, "test_service")) {
+        try (var sender = HttpRequestSenderTests.createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -206,7 +206,7 @@ public void testExecute_ThrowsException() {
     public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = HttpRequestSenderTests.createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java
index a55b3c5f5030c..7e98b9b31f6ed 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/googleaistudio/GoogleAiStudioEmbeddingsActionTests.java
@@ -79,7 +79,7 @@ public void testExecute_ReturnsSuccessfulResponse() throws IOException {
         var input = "input";
         var senderFactory = new HttpRequestSender.Factory(createWithEmptySettings(threadPool), clientManager, mockClusterServiceEmpty());
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = senderFactory.createSender()) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionCreatorTests.java
index fceea8810f6c2..b3ec565b3146a 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionCreatorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceActionCreatorTests.java
@@ -42,6 +42,7 @@
 import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
 import static org.elasticsearch.xpack.inference.external.http.retry.RetrySettingsTests.buildSettingsWithRetryFields;
+import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender;
 import static org.elasticsearch.xpack.inference.logging.ThrottlerManagerTests.mockThrottlerManager;
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
 import static org.hamcrest.Matchers.contains;
@@ -75,7 +76,7 @@ public void shutdown() throws IOException {
     public void testExecute_ReturnsSuccessfulResponse_ForElserAction() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -131,7 +132,7 @@ public void testSend_FailsFromInvalidResponseFormat_ForElserAction() throws IOEx
         );
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager, settings);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -187,7 +188,7 @@ public void testSend_FailsFromInvalidResponseFormat_ForElserAction() throws IOEx
     public void testExecute_ReturnsSuccessfulResponse_ForEmbeddingsAction() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -239,7 +240,7 @@ public void testSend_FailsFromInvalidResponseFormat_ForEmbeddingsAction() throws
         );
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager, settings);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             // this will fail because the only valid formats are {"embeddings": [[...]]} or [[...]]
@@ -292,7 +293,7 @@ public void testSend_FailsFromInvalidResponseFormat_ForEmbeddingsAction() throws
     public void testExecute_ReturnsSuccessfulResponse_AfterTruncating() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJsonContentTooLarge = """
@@ -357,7 +358,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating() throws IOExc
     public void testExecute_TruncatesInputBeforeSending() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java
index 496238eaad0e4..b6d7eb673b7f0 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiActionCreatorTests.java
@@ -38,6 +38,7 @@
 import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
 import static org.elasticsearch.xpack.inference.external.http.retry.RetrySettingsTests.buildSettingsWithRetryFields;
+import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender;
 import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiUtils.ORGANIZATION_HEADER;
 import static org.elasticsearch.xpack.inference.results.ChatCompletionResultsTests.buildExpectationCompletion;
 import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat;
@@ -74,7 +75,7 @@ public void shutdown() throws IOException {
     public void testCreate_OpenAiEmbeddingsModel() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -127,7 +128,7 @@ public void testCreate_OpenAiEmbeddingsModel() throws IOException {
     public void testCreate_OpenAiEmbeddingsModel_WithoutUser() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -179,7 +180,7 @@ public void testCreate_OpenAiEmbeddingsModel_WithoutUser() throws IOException {
     public void testCreate_OpenAiEmbeddingsModel_WithoutOrganization() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -238,7 +239,7 @@ public void testCreate_OpenAiEmbeddingsModel_FailsFromInvalidResponseFormat() th
         );
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager, settings);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -292,7 +293,7 @@ public void testCreate_OpenAiEmbeddingsModel_FailsFromInvalidResponseFormat() th
     public void testCreate_OpenAiChatCompletionModel() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -355,7 +356,7 @@ public void testCreate_OpenAiChatCompletionModel() throws IOException {
     public void testCreate_OpenAiChatCompletionModel_WithoutUser() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -417,7 +418,7 @@ public void testCreate_OpenAiChatCompletionModel_WithoutUser() throws IOExceptio
     public void testCreate_OpenAiChatCompletionModel_WithoutOrganization() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -486,7 +487,7 @@ public void testCreate_OpenAiChatCompletionModel_FailsFromInvalidResponseFormat(
         );
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager, settings);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -552,7 +553,7 @@ public void testCreate_OpenAiChatCompletionModel_FailsFromInvalidResponseFormat(
     public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From413StatusCode() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             var contentTooLargeErrorMessage =
@@ -635,7 +636,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From413StatusC
     public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From400StatusCode() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             var contentTooLargeErrorMessage =
@@ -718,7 +719,7 @@ public void testExecute_ReturnsSuccessfulResponse_AfterTruncating_From400StatusC
     public void testExecute_TruncatesInputBeforeSending() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java
index 914ff12db259a..42b062667f770 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionActionTests.java
@@ -43,6 +43,7 @@
 import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty;
 import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap;
 import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl;
+import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender;
 import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiUtils.ORGANIZATION_HEADER;
 import static org.elasticsearch.xpack.inference.results.ChatCompletionResultsTests.buildExpectationCompletion;
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
@@ -80,7 +81,7 @@ public void shutdown() throws IOException {
     public void testExecute_ReturnsSuccessfulResponse() throws IOException {
         var senderFactory = new HttpRequestSender.Factory(createWithEmptySettings(threadPool), clientManager, mockClusterServiceEmpty());
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -234,7 +235,7 @@ public void testExecute_ThrowsExceptionWithNullUrl() {
     public void testExecute_ThrowsException_WhenInputIsGreaterThanOne() throws IOException {
         var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java
index 15b7417912ef5..03c0b4d146b2e 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsActionTests.java
@@ -79,7 +79,7 @@ public void testExecute_ReturnsSuccessfulResponse() throws IOException {
             mockClusterServiceEmpty()
         );
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = senderFactory.createSender()) {
             sender.start();
 
             String responseJson = """
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManagerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManagerTests.java
new file mode 100644
index 0000000000000..03838896b879d
--- /dev/null
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManagerTests.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.inference.external.http.sender;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.inference.InferenceServiceResults;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.mockito.Mockito.mock;
+
+public class BaseRequestManagerTests extends ESTestCase {
+    public void testRateLimitGrouping_DifferentObjectReferences_HaveSameGroup() {
+        int val1 = 1;
+        int val2 = 1;
+
+        var manager1 = new BaseRequestManager(mock(ThreadPool.class), "id", val1, new RateLimitSettings(1)) {
+            @Override
+            public void execute(
+                String query,
+                List<String> input,
+                RequestSender requestSender,
+                Supplier<Boolean> hasRequestCompletedFunction,
+                ActionListener<InferenceServiceResults> listener
+            ) {
+
+            }
+        };
+
+        var manager2 = new BaseRequestManager(mock(ThreadPool.class), "id", val2, new RateLimitSettings(1)) {
+            @Override
+            public void execute(
+                String query,
+                List<String> input,
+                RequestSender requestSender,
+                Supplier<Boolean> hasRequestCompletedFunction,
+                ActionListener<InferenceServiceResults> listener
+            ) {
+
+            }
+        };
+
+        assertThat(manager1.rateLimitGrouping(), is(manager2.rateLimitGrouping()));
+    }
+
+    public void testRateLimitGrouping_DifferentSettings_HaveDifferentGroup() {
+        int val1 = 1;
+
+        var manager1 = new BaseRequestManager(mock(ThreadPool.class), "id", val1, new RateLimitSettings(1)) {
+            @Override
+            public void execute(
+                String query,
+                List<String> input,
+                RequestSender requestSender,
+                Supplier<Boolean> hasRequestCompletedFunction,
+                ActionListener<InferenceServiceResults> listener
+            ) {
+
+            }
+        };
+
+        var manager2 = new BaseRequestManager(mock(ThreadPool.class), "id", val1, new RateLimitSettings(2)) {
+            @Override
+            public void execute(
+                String query,
+                List<String> input,
+                RequestSender requestSender,
+                Supplier<Boolean> hasRequestCompletedFunction,
+                ActionListener<InferenceServiceResults> listener
+            ) {
+
+            }
+        };
+
+        assertThat(manager1.rateLimitGrouping(), not(manager2.rateLimitGrouping()));
+    }
+
+    public void testRateLimitGrouping_DifferentSettingsTimeUnit_HaveDifferentGroup() {
+        int val1 = 1;
+
+        var manager1 = new BaseRequestManager(mock(ThreadPool.class), "id", val1, new RateLimitSettings(1, TimeUnit.MILLISECONDS)) {
+            @Override
+            public void execute(
+                String query,
+                List<String> input,
+                RequestSender requestSender,
+                Supplier<Boolean> hasRequestCompletedFunction,
+                ActionListener<InferenceServiceResults> listener
+            ) {
+
+            }
+        };
+
+        var manager2 = new BaseRequestManager(mock(ThreadPool.class), "id", val1, new RateLimitSettings(1, TimeUnit.DAYS)) {
+            @Override
+            public void execute(
+                String query,
+                List<String> input,
+                RequestSender requestSender,
+                Supplier<Boolean> hasRequestCompletedFunction,
+                ActionListener<InferenceServiceResults> listener
+            ) {
+
+            }
+        };
+
+        assertThat(manager1.rateLimitGrouping(), not(manager2.rateLimitGrouping()));
+    }
+}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java
index 368745b310884..2b8b5f178b3de 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java
@@ -79,7 +79,7 @@ public void shutdown() throws IOException, InterruptedException {
     public void testCreateSender_SendsRequestAndReceivesResponse() throws Exception {
         var senderFactory = createSenderFactory(clientManager, threadRef);
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = createSender(senderFactory)) {
             sender.start();
 
             String responseJson = """
@@ -135,11 +135,11 @@ public void testHttpRequestSender_Throws_WhenCallingSendBeforeStart() throws Exc
             mockClusterServiceEmpty()
         );
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = senderFactory.createSender()) {
             PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
             var thrownException = expectThrows(
                 AssertionError.class,
-                () -> sender.send(ExecutableRequestCreatorTests.createMock(), new DocumentsOnlyInput(List.of()), null, listener)
+                () -> sender.send(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), null, listener)
             );
             assertThat(thrownException.getMessage(), is("call start() before sending a request"));
         }
@@ -155,17 +155,12 @@ public void testHttpRequestSender_Throws_WhenATimeoutOccurs() throws Exception {
             mockClusterServiceEmpty()
         );
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = senderFactory.createSender()) {
             assertThat(sender, instanceOf(HttpRequestSender.class));
             sender.start();
 
             PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
-            sender.send(
-                ExecutableRequestCreatorTests.createMock(),
-                new DocumentsOnlyInput(List.of()),
-                TimeValue.timeValueNanos(1),
-                listener
-            );
+            sender.send(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), TimeValue.timeValueNanos(1), listener);
 
             var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT));
 
@@ -186,16 +181,11 @@ public void testHttpRequestSenderWithTimeout_Throws_WhenATimeoutOccurs() throws
             mockClusterServiceEmpty()
         );
 
-        try (var sender = senderFactory.createSender("test_service")) {
+        try (var sender = senderFactory.createSender()) {
             sender.start();
 
             PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
-            sender.send(
-                ExecutableRequestCreatorTests.createMock(),
-                new DocumentsOnlyInput(List.of()),
-                TimeValue.timeValueNanos(1),
-                listener
-            );
+            sender.send(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), TimeValue.timeValueNanos(1), listener);
 
             var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT));
 
@@ -220,6 +210,7 @@ private static HttpRequestSender.Factory createSenderFactory(HttpClientManager c
         when(mockThreadPool.executor(anyString())).thenReturn(mockExecutorService);
         when(mockThreadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
         when(mockThreadPool.schedule(any(Runnable.class), any(), any())).thenReturn(mock(Scheduler.ScheduledCancellable.class));
+        when(mockThreadPool.scheduleWithFixedDelay(any(Runnable.class), any(), any())).thenReturn(mock(Scheduler.Cancellable.class));
 
         return new HttpRequestSender.Factory(
             ServiceComponentsTests.createWithEmptySettings(mockThreadPool),
@@ -248,7 +239,7 @@ public static HttpRequestSender.Factory createSenderFactory(
         );
     }
 
-    public static Sender createSenderWithSingleRequestManager(HttpRequestSender.Factory factory, String serviceName) {
-        return factory.createSender(serviceName);
+    public static Sender createSender(HttpRequestSender.Factory factory) {
+        return factory.createSender();
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettingsTests.java
index c0c0bdd49f617..489b502c04110 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceSettingsTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 
 import static org.elasticsearch.xpack.inference.Utils.mockClusterService;
 
@@ -18,12 +19,23 @@ public static RequestExecutorServiceSettings createRequestExecutorServiceSetting
     }
 
     public static RequestExecutorServiceSettings createRequestExecutorServiceSettings(@Nullable Integer queueCapacity) {
+        return createRequestExecutorServiceSettings(queueCapacity, null);
+    }
+
+    public static RequestExecutorServiceSettings createRequestExecutorServiceSettings(
+        @Nullable Integer queueCapacity,
+        @Nullable TimeValue staleDuration
+    ) {
         var settingsBuilder = Settings.builder();
 
         if (queueCapacity != null) {
             settingsBuilder.put(RequestExecutorServiceSettings.TASK_QUEUE_CAPACITY_SETTING.getKey(), queueCapacity);
         }
 
+        if (staleDuration != null) {
+            settingsBuilder.put(RequestExecutorServiceSettings.RATE_LIMIT_GROUP_STALE_DURATION_SETTING.getKey(), staleDuration);
+        }
+
         return createRequestExecutorServiceSettings(settingsBuilder.build());
     }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java
index ff88ba221d985..9a45e10007643 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java
@@ -18,13 +18,19 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.inference.common.RateLimiter;
+import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
 import org.elasticsearch.xpack.inference.external.http.retry.RetryingHttpSender;
 import org.junit.After;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CountDownLatch;
@@ -42,10 +48,13 @@
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
 
 public class RequestExecutorServiceTests extends ESTestCase {
@@ -70,7 +79,7 @@ public void testQueueSize_IsEmpty() {
 
     public void testQueueSize_IsOne() {
         var service = createRequestExecutorServiceWithMocks();
-        service.execute(ExecutableRequestCreatorTests.createMock(), new DocumentsOnlyInput(List.of()), null, new PlainActionFuture<>());
+        service.execute(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), null, new PlainActionFuture<>());
 
         assertThat(service.queueSize(), is(1));
     }
@@ -92,7 +101,20 @@ public void testIsTerminated_IsTrue() throws InterruptedException {
         assertTrue(service.isTerminated());
     }
 
-    public void testIsTerminated_AfterStopFromSeparateThread() throws Exception {
+    public void testCallingStartTwice_ThrowsAssertionException() throws InterruptedException {
+        var latch = new CountDownLatch(1);
+        var service = createRequestExecutorService(latch, mock(RetryingHttpSender.class));
+
+        service.shutdown();
+        service.start();
+        latch.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS);
+
+        assertTrue(service.isTerminated());
+        var exception = expectThrows(AssertionError.class, service::start);
+        assertThat(exception.getMessage(), is("start() can only be called once"));
+    }
+
+    public void testIsTerminated_AfterStopFromSeparateThread() {
         var waitToShutdown = new CountDownLatch(1);
         var waitToReturnFromSend = new CountDownLatch(1);
 
@@ -127,41 +149,48 @@ public void testIsTerminated_AfterStopFromSeparateThread() throws Exception {
         assertTrue(service.isTerminated());
     }
 
-    public void testSend_AfterShutdown_Throws() {
+    public void testExecute_AfterShutdown_Throws() {
         var service = createRequestExecutorServiceWithMocks();
 
         service.shutdown();
 
+        var requestManager = RequestManagerTests.createMock("id");
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        service.execute(ExecutableRequestCreatorTests.createMock(), new DocumentsOnlyInput(List.of()), null, listener);
+        service.execute(requestManager, new DocumentsOnlyInput(List.of()), null, listener);
 
         var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT));
 
         assertThat(
             thrownException.getMessage(),
-            is("Failed to enqueue task because the http executor service [test_service] has already shutdown")
+            is(
+                Strings.format(
+                    "Failed to enqueue task for inference id [id] because the request service [%s] has already shutdown",
+                    requestManager.rateLimitGrouping().hashCode()
+                )
+            )
         );
         assertTrue(thrownException.isExecutorShutdown());
     }
 
-    public void testSend_Throws_WhenQueueIsFull() {
-        var service = new RequestExecutorService(
-            "test_service",
-            threadPool,
-            null,
-            createRequestExecutorServiceSettings(1),
-            new SingleRequestManager(mock(RetryingHttpSender.class))
-        );
+    public void testExecute_Throws_WhenQueueIsFull() {
+        var service = new RequestExecutorService(threadPool, null, createRequestExecutorServiceSettings(1), mock(RetryingHttpSender.class));
 
-        service.execute(ExecutableRequestCreatorTests.createMock(), new DocumentsOnlyInput(List.of()), null, new PlainActionFuture<>());
+        service.execute(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), null, new PlainActionFuture<>());
+
+        var requestManager = RequestManagerTests.createMock("id");
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        service.execute(ExecutableRequestCreatorTests.createMock(), new DocumentsOnlyInput(List.of()), null, listener);
+        service.execute(requestManager, new DocumentsOnlyInput(List.of()), null, listener);
 
         var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT));
 
         assertThat(
             thrownException.getMessage(),
-            is("Failed to execute task because the http executor service [test_service] queue is full")
+            is(
+                Strings.format(
+                    "Failed to execute task for inference id [id] because the request service [%s] queue is full",
+                    requestManager.rateLimitGrouping().hashCode()
+                )
+            )
         );
         assertFalse(thrownException.isExecutorShutdown());
     }
@@ -203,16 +232,11 @@ public void testShutdown_AllowsMultipleCalls() {
         assertTrue(service.isShutdown());
     }
 
-    public void testSend_CallsOnFailure_WhenRequestTimesOut() {
+    public void testExecute_CallsOnFailure_WhenRequestTimesOut() {
         var service = createRequestExecutorServiceWithMocks();
 
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        service.execute(
-            ExecutableRequestCreatorTests.createMock(),
-            new DocumentsOnlyInput(List.of()),
-            TimeValue.timeValueNanos(1),
-            listener
-        );
+        service.execute(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), TimeValue.timeValueNanos(1), listener);
 
         var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT));
 
@@ -222,7 +246,7 @@ public void testSend_CallsOnFailure_WhenRequestTimesOut() {
         );
     }
 
-    public void testSend_PreservesThreadContext() throws InterruptedException, ExecutionException, TimeoutException {
+    public void testExecute_PreservesThreadContext() throws InterruptedException, ExecutionException, TimeoutException {
         var headerKey = "not empty";
         var headerValue = "value";
 
@@ -270,7 +294,7 @@ public void onFailure(Exception e) {
             }
         };
 
-        service.execute(ExecutableRequestCreatorTests.createMock(requestSender), new DocumentsOnlyInput(List.of()), null, listener);
+        service.execute(RequestManagerTests.createMock(requestSender), new DocumentsOnlyInput(List.of()), null, listener);
 
         Future<?> executorTermination = submitShutdownRequest(waitToShutdown, waitToReturnFromSend, service);
 
@@ -280,11 +304,12 @@ public void onFailure(Exception e) {
         finishedOnResponse.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS);
     }
 
-    public void testSend_NotifiesTasksOfShutdown() {
+    public void testExecute_NotifiesTasksOfShutdown() {
         var service = createRequestExecutorServiceWithMocks();
 
+        var requestManager = RequestManagerTests.createMock(mock(RequestSender.class), "id");
         var listener = new PlainActionFuture<InferenceServiceResults>();
-        service.execute(ExecutableRequestCreatorTests.createMock(), new DocumentsOnlyInput(List.of()), null, listener);
+        service.execute(requestManager, new DocumentsOnlyInput(List.of()), null, listener);
 
         service.shutdown();
         service.start();
@@ -293,47 +318,62 @@ public void testSend_NotifiesTasksOfShutdown() {
 
         assertThat(
             thrownException.getMessage(),
-            is("Failed to send request, queue service [test_service] has shutdown prior to executing request")
+            is(
+                Strings.format(
+                    "Failed to send request, request service [%s] for inference id [id] has shutdown prior to executing request",
+                    requestManager.rateLimitGrouping().hashCode()
+                )
+            )
         );
         assertTrue(thrownException.isExecutorShutdown());
         assertTrue(service.isTerminated());
     }
 
-    public void testQueueTake_DoesNotCauseServiceToTerminate_WhenItThrows() throws InterruptedException {
+    public void testQueuePoll_DoesNotCauseServiceToTerminate_WhenItThrows() throws InterruptedException {
         @SuppressWarnings("unchecked")
         BlockingQueue<RejectableTask> queue = mock(LinkedBlockingQueue.class);
 
+        var requestSender = mock(RetryingHttpSender.class);
+
         var service = new RequestExecutorService(
-            getTestName(),
             threadPool,
             mockQueueCreator(queue),
             null,
             createRequestExecutorServiceSettingsEmpty(),
-            new SingleRequestManager(mock(RetryingHttpSender.class))
+            requestSender,
+            Clock.systemUTC(),
+            RequestExecutorService.DEFAULT_SLEEPER,
+            RequestExecutorService.DEFAULT_RATE_LIMIT_CREATOR
         );
 
-        when(queue.take()).thenThrow(new ElasticsearchException("failed")).thenAnswer(invocation -> {
+        PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+        var requestManager = RequestManagerTests.createMock(requestSender, "id");
+        service.execute(requestManager, new DocumentsOnlyInput(List.of()), null, listener);
+
+        when(queue.poll()).thenThrow(new ElasticsearchException("failed")).thenAnswer(invocation -> {
             service.shutdown();
             return null;
         });
         service.start();
 
         assertTrue(service.isTerminated());
-        verify(queue, times(2)).take();
     }
 
-    public void testQueueTake_ThrowingInterruptedException_TerminatesService() throws Exception {
+    public void testSleep_ThrowingInterruptedException_TerminatesService() throws Exception {
         @SuppressWarnings("unchecked")
         BlockingQueue<RejectableTask> queue = mock(LinkedBlockingQueue.class);
-        when(queue.take()).thenThrow(new InterruptedException("failed"));
+        var sleeper = mock(RequestExecutorService.Sleeper.class);
+        doThrow(new InterruptedException("failed")).when(sleeper).sleep(any());
 
         var service = new RequestExecutorService(
-            getTestName(),
             threadPool,
             mockQueueCreator(queue),
             null,
             createRequestExecutorServiceSettingsEmpty(),
-            new SingleRequestManager(mock(RetryingHttpSender.class))
+            mock(RetryingHttpSender.class),
+            Clock.systemUTC(),
+            sleeper,
+            RequestExecutorService.DEFAULT_RATE_LIMIT_CREATOR
         );
 
         Future<?> executorTermination = threadPool.generic().submit(() -> {
@@ -347,66 +387,30 @@ public void testQueueTake_ThrowingInterruptedException_TerminatesService() throw
         executorTermination.get(TIMEOUT.millis(), TimeUnit.MILLISECONDS);
 
         assertTrue(service.isTerminated());
-        verify(queue, times(1)).take();
-    }
-
-    public void testQueueTake_RejectsTask_WhenServiceShutsDown() throws Exception {
-        var mockTask = mock(RejectableTask.class);
-        @SuppressWarnings("unchecked")
-        BlockingQueue<RejectableTask> queue = mock(LinkedBlockingQueue.class);
-
-        var service = new RequestExecutorService(
-            "test_service",
-            threadPool,
-            mockQueueCreator(queue),
-            null,
-            createRequestExecutorServiceSettingsEmpty(),
-            new SingleRequestManager(mock(RetryingHttpSender.class))
-        );
-
-        doAnswer(invocation -> {
-            service.shutdown();
-            return mockTask;
-        }).doReturn(new NoopTask()).when(queue).take();
-
-        service.start();
-
-        assertTrue(service.isTerminated());
-        verify(queue, times(1)).take();
-
-        ArgumentCaptor<Exception> argument = ArgumentCaptor.forClass(Exception.class);
-        verify(mockTask, times(1)).onRejection(argument.capture());
-        assertThat(argument.getValue(), instanceOf(EsRejectedExecutionException.class));
-        assertThat(
-            argument.getValue().getMessage(),
-            is("Failed to send request, queue service [test_service] has shutdown prior to executing request")
-        );
-
-        var rejectionException = (EsRejectedExecutionException) argument.getValue();
-        assertTrue(rejectionException.isExecutorShutdown());
     }
 
     public void testChangingCapacity_SetsCapacityToTwo() throws ExecutionException, InterruptedException, TimeoutException {
         var requestSender = mock(RetryingHttpSender.class);
 
         var settings = createRequestExecutorServiceSettings(1);
-        var service = new RequestExecutorService("test_service", threadPool, null, settings, new SingleRequestManager(requestSender));
+        var service = new RequestExecutorService(threadPool, null, settings, requestSender);
 
-        service.execute(
-            ExecutableRequestCreatorTests.createMock(requestSender),
-            new DocumentsOnlyInput(List.of()),
-            null,
-            new PlainActionFuture<>()
-        );
+        service.execute(RequestManagerTests.createMock(requestSender), new DocumentsOnlyInput(List.of()), null, new PlainActionFuture<>());
         assertThat(service.queueSize(), is(1));
 
         PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
-        service.execute(ExecutableRequestCreatorTests.createMock(requestSender), new DocumentsOnlyInput(List.of()), null, listener);
+        var requestManager = RequestManagerTests.createMock(requestSender, "id");
+        service.execute(requestManager, new DocumentsOnlyInput(List.of()), null, listener);
 
         var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT));
         assertThat(
             thrownException.getMessage(),
-            is("Failed to execute task because the http executor service [test_service] queue is full")
+            is(
+                Strings.format(
+                    "Failed to execute task for inference id [id] because the request service [%s] queue is full",
+                    requestManager.rateLimitGrouping().hashCode()
+                )
+            )
         );
 
         settings.setQueueCapacity(2);
@@ -426,7 +430,7 @@ public void testChangingCapacity_SetsCapacityToTwo() throws ExecutionException,
 
         executorTermination.get(TIMEOUT.millis(), TimeUnit.MILLISECONDS);
         assertTrue(service.isTerminated());
-        assertThat(service.remainingQueueCapacity(), is(2));
+        assertThat(service.remainingQueueCapacity(requestManager), is(2));
     }
 
     public void testChangingCapacity_DoesNotRejectsOverflowTasks_BecauseOfQueueFull() throws ExecutionException, InterruptedException,
@@ -434,23 +438,24 @@ public void testChangingCapacity_DoesNotRejectsOverflowTasks_BecauseOfQueueFull(
         var requestSender = mock(RetryingHttpSender.class);
 
         var settings = createRequestExecutorServiceSettings(3);
-        var service = new RequestExecutorService("test_service", threadPool, null, settings, new SingleRequestManager(requestSender));
+        var service = new RequestExecutorService(threadPool, null, settings, requestSender);
 
         service.execute(
-            ExecutableRequestCreatorTests.createMock(requestSender),
+            RequestManagerTests.createMock(requestSender, "id"),
             new DocumentsOnlyInput(List.of()),
             null,
             new PlainActionFuture<>()
         );
         service.execute(
-            ExecutableRequestCreatorTests.createMock(requestSender),
+            RequestManagerTests.createMock(requestSender, "id"),
             new DocumentsOnlyInput(List.of()),
             null,
             new PlainActionFuture<>()
         );
 
         PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
-        service.execute(ExecutableRequestCreatorTests.createMock(requestSender), new DocumentsOnlyInput(List.of()), null, listener);
+        var requestManager = RequestManagerTests.createMock(requestSender, "id");
+        service.execute(requestManager, new DocumentsOnlyInput(List.of()), null, listener);
         assertThat(service.queueSize(), is(3));
 
         settings.setQueueCapacity(1);
@@ -470,7 +475,7 @@ public void testChangingCapacity_DoesNotRejectsOverflowTasks_BecauseOfQueueFull(
 
         executorTermination.get(TIMEOUT.millis(), TimeUnit.MILLISECONDS);
         assertTrue(service.isTerminated());
-        assertThat(service.remainingQueueCapacity(), is(1));
+        assertThat(service.remainingQueueCapacity(requestManager), is(1));
         assertThat(service.queueSize(), is(0));
 
         var thrownException = expectThrows(
@@ -479,7 +484,12 @@ public void testChangingCapacity_DoesNotRejectsOverflowTasks_BecauseOfQueueFull(
         );
         assertThat(
             thrownException.getMessage(),
-            is("Failed to send request, queue service [test_service] has shutdown prior to executing request")
+            is(
+                Strings.format(
+                    "Failed to send request, request service [%s] for inference id [id] has shutdown prior to executing request",
+                    requestManager.rateLimitGrouping().hashCode()
+                )
+            )
         );
         assertTrue(thrownException.isExecutorShutdown());
     }
@@ -489,23 +499,24 @@ public void testChangingCapacity_ToZero_SetsQueueCapacityToUnbounded() throws IO
         var requestSender = mock(RetryingHttpSender.class);
 
         var settings = createRequestExecutorServiceSettings(1);
-        var service = new RequestExecutorService("test_service", threadPool, null, settings, new SingleRequestManager(requestSender));
+        var service = new RequestExecutorService(threadPool, null, settings, requestSender);
+        var requestManager = RequestManagerTests.createMock(requestSender);
 
-        service.execute(
-            ExecutableRequestCreatorTests.createMock(requestSender),
-            new DocumentsOnlyInput(List.of()),
-            null,
-            new PlainActionFuture<>()
-        );
+        service.execute(requestManager, new DocumentsOnlyInput(List.of()), null, new PlainActionFuture<>());
         assertThat(service.queueSize(), is(1));
 
         PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
-        service.execute(ExecutableRequestCreatorTests.createMock(requestSender), new DocumentsOnlyInput(List.of()), null, listener);
+        service.execute(RequestManagerTests.createMock(requestSender, "id"), new DocumentsOnlyInput(List.of()), null, listener);
 
         var thrownException = expectThrows(EsRejectedExecutionException.class, () -> listener.actionGet(TIMEOUT));
         assertThat(
             thrownException.getMessage(),
-            is("Failed to execute task because the http executor service [test_service] queue is full")
+            is(
+                Strings.format(
+                    "Failed to execute task for inference id [id] because the request service [%s] queue is full",
+                    requestManager.rateLimitGrouping().hashCode()
+                )
+            )
         );
 
         settings.setQueueCapacity(0);
@@ -525,7 +536,133 @@ public void testChangingCapacity_ToZero_SetsQueueCapacityToUnbounded() throws IO
 
         executorTermination.get(TIMEOUT.millis(), TimeUnit.MILLISECONDS);
         assertTrue(service.isTerminated());
-        assertThat(service.remainingQueueCapacity(), is(Integer.MAX_VALUE));
+        assertThat(service.remainingQueueCapacity(requestManager), is(Integer.MAX_VALUE));
+    }
+
+    public void testDoesNotExecuteTask_WhenCannotReserveTokens() {
+        var mockRateLimiter = mock(RateLimiter.class);
+        RequestExecutorService.RateLimiterCreator rateLimiterCreator = (a, b, c) -> mockRateLimiter;
+
+        var requestSender = mock(RetryingHttpSender.class);
+        var settings = createRequestExecutorServiceSettings(1);
+        var service = new RequestExecutorService(
+            threadPool,
+            RequestExecutorService.DEFAULT_QUEUE_CREATOR,
+            null,
+            settings,
+            requestSender,
+            Clock.systemUTC(),
+            RequestExecutorService.DEFAULT_SLEEPER,
+            rateLimiterCreator
+        );
+        var requestManager = RequestManagerTests.createMock(requestSender);
+
+        PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+        service.execute(requestManager, new DocumentsOnlyInput(List.of()), null, listener);
+
+        doAnswer(invocation -> {
+            service.shutdown();
+            return TimeValue.timeValueDays(1);
+        }).when(mockRateLimiter).timeToReserve(anyInt());
+
+        service.start();
+
+        verifyNoInteractions(requestSender);
+    }
+
+    public void testDoesNotExecuteTask_WhenCannotReserveTokens_AndThenCanReserve_AndExecutesTask() {
+        var mockRateLimiter = mock(RateLimiter.class);
+        when(mockRateLimiter.reserve(anyInt())).thenReturn(TimeValue.timeValueDays(0));
+
+        RequestExecutorService.RateLimiterCreator rateLimiterCreator = (a, b, c) -> mockRateLimiter;
+
+        var requestSender = mock(RetryingHttpSender.class);
+        var settings = createRequestExecutorServiceSettings(1);
+        var service = new RequestExecutorService(
+            threadPool,
+            RequestExecutorService.DEFAULT_QUEUE_CREATOR,
+            null,
+            settings,
+            requestSender,
+            Clock.systemUTC(),
+            RequestExecutorService.DEFAULT_SLEEPER,
+            rateLimiterCreator
+        );
+        var requestManager = RequestManagerTests.createMock(requestSender);
+
+        PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+        service.execute(requestManager, new DocumentsOnlyInput(List.of()), null, listener);
+
+        when(mockRateLimiter.timeToReserve(anyInt())).thenReturn(TimeValue.timeValueDays(1)).thenReturn(TimeValue.timeValueDays(0));
+
+        doAnswer(invocation -> {
+            service.shutdown();
+            return Void.TYPE;
+        }).when(requestSender).send(any(), any(), any(), any(), any(), any());
+
+        service.start();
+
+        verify(requestSender, times(1)).send(any(), any(), any(), any(), any(), any());
+    }
+
+    public void testRemovesRateLimitGroup_AfterStaleDuration() {
+        var now = Instant.now();
+        var clock = mock(Clock.class);
+        when(clock.instant()).thenReturn(now);
+
+        var requestSender = mock(RetryingHttpSender.class);
+        var settings = createRequestExecutorServiceSettings(2, TimeValue.timeValueDays(1));
+        var service = new RequestExecutorService(
+            threadPool,
+            RequestExecutorService.DEFAULT_QUEUE_CREATOR,
+            null,
+            settings,
+            requestSender,
+            clock,
+            RequestExecutorService.DEFAULT_SLEEPER,
+            RequestExecutorService.DEFAULT_RATE_LIMIT_CREATOR
+        );
+        var requestManager = RequestManagerTests.createMock(requestSender, "id1");
+
+        PlainActionFuture<InferenceServiceResults> listener = new PlainActionFuture<>();
+        service.execute(requestManager, new DocumentsOnlyInput(List.of()), null, listener);
+
+        assertThat(service.numberOfRateLimitGroups(), is(1));
+        // the time is moved to after the stale duration, so now we should remove this grouping
+        when(clock.instant()).thenReturn(now.plus(Duration.ofDays(2)));
+        service.removeStaleGroupings();
+        assertThat(service.numberOfRateLimitGroups(), is(0));
+
+        var requestManager2 = RequestManagerTests.createMock(requestSender, "id2");
+        service.execute(requestManager2, new DocumentsOnlyInput(List.of()), null, listener);
+
+        assertThat(service.numberOfRateLimitGroups(), is(1));
+    }
+
+    public void testStartsCleanupThread() {
+        var mockThreadPool = mock(ThreadPool.class);
+
+        when(mockThreadPool.scheduleWithFixedDelay(any(Runnable.class), any(), any())).thenReturn(mock(Scheduler.Cancellable.class));
+
+        var requestSender = mock(RetryingHttpSender.class);
+        var settings = createRequestExecutorServiceSettings(2, TimeValue.timeValueDays(1));
+        var service = new RequestExecutorService(
+            mockThreadPool,
+            RequestExecutorService.DEFAULT_QUEUE_CREATOR,
+            null,
+            settings,
+            requestSender,
+            Clock.systemUTC(),
+            RequestExecutorService.DEFAULT_SLEEPER,
+            RequestExecutorService.DEFAULT_RATE_LIMIT_CREATOR
+        );
+
+        service.shutdown();
+        service.start();
+
+        ArgumentCaptor<TimeValue> argument = ArgumentCaptor.forClass(TimeValue.class);
+        verify(mockThreadPool, times(1)).scheduleWithFixedDelay(any(Runnable.class), argument.capture(), any());
+        assertThat(argument.getValue(), is(TimeValue.timeValueDays(1)));
     }
 
     private Future<?> submitShutdownRequest(
@@ -552,12 +689,6 @@ private RequestExecutorService createRequestExecutorServiceWithMocks() {
     }
 
     private RequestExecutorService createRequestExecutorService(@Nullable CountDownLatch startupLatch, RetryingHttpSender requestSender) {
-        return new RequestExecutorService(
-            "test_service",
-            threadPool,
-            startupLatch,
-            createRequestExecutorServiceSettingsEmpty(),
-            new SingleRequestManager(requestSender)
-        );
+        return new RequestExecutorService(threadPool, startupLatch, createRequestExecutorServiceSettingsEmpty(), requestSender);
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableRequestCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java
similarity index 56%
rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableRequestCreatorTests.java
rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java
index 31297ed432ef5..291de740aca34 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableRequestCreatorTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.inference.external.http.retry.RequestSender;
 import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler;
 import org.elasticsearch.xpack.inference.external.request.RequestTests;
+import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyList;
@@ -21,34 +22,47 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class ExecutableRequestCreatorTests {
+public class RequestManagerTests {
     public static RequestManager createMock() {
-        var mockCreator = mock(RequestManager.class);
-        when(mockCreator.create(any(), anyList(), any(), any(), any(), any())).thenReturn(() -> {});
+        return createMock(mock(RequestSender.class));
+    }
 
-        return mockCreator;
+    public static RequestManager createMock(String inferenceEntityId) {
+        return createMock(mock(RequestSender.class), inferenceEntityId);
     }
 
     public static RequestManager createMock(RequestSender requestSender) {
-        return createMock(requestSender, "id");
+        return createMock(requestSender, "id", new RateLimitSettings(1));
+    }
+
+    public static RequestManager createMock(RequestSender requestSender, String inferenceEntityId) {
+        return createMock(requestSender, inferenceEntityId, new RateLimitSettings(1));
     }
 
-    public static RequestManager createMock(RequestSender requestSender, String modelId) {
-        var mockCreator = mock(RequestManager.class);
+    public static RequestManager createMock(RequestSender requestSender, String inferenceEntityId, RateLimitSettings settings) {
+        var mockManager = mock(RequestManager.class);
 
         doAnswer(invocation -> {
             @SuppressWarnings("unchecked")
-            ActionListener<InferenceServiceResults> listener = (ActionListener<InferenceServiceResults>) invocation.getArguments()[5];
-            return (Runnable) () -> requestSender.send(
+            ActionListener<InferenceServiceResults> listener = (ActionListener<InferenceServiceResults>) invocation.getArguments()[4];
+            requestSender.send(
                 mock(Logger.class),
-                RequestTests.mockRequest(modelId),
+                RequestTests.mockRequest(inferenceEntityId),
                 HttpClientContext.create(),
                 () -> false,
                 mock(ResponseHandler.class),
                 listener
             );
-        }).when(mockCreator).create(any(), anyList(), any(), any(), any(), any());
 
-        return mockCreator;
+            return Void.TYPE;
+        }).when(mockManager).execute(any(), anyList(), any(), any(), any());
+
+        // just return something consistent so the hashing works
+        when(mockManager.rateLimitGrouping()).thenReturn(inferenceEntityId);
+
+        when(mockManager.rateLimitSettings()).thenReturn(settings);
+        when(mockManager.inferenceEntityId()).thenReturn(inferenceEntityId);
+
+        return mockManager;
     }
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManagerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManagerTests.java
deleted file mode 100644
index 55965bc2354d3..0000000000000
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManagerTests.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.inference.external.http.sender;
-
-import org.apache.http.client.protocol.HttpClientContext;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.inference.external.http.retry.RetryingHttpSender;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verifyNoInteractions;
-import static org.mockito.Mockito.when;
-
-public class SingleRequestManagerTests extends ESTestCase {
-    public void testExecute_DoesNotCallRequestCreatorCreate_WhenInputIsNull() {
-        var requestCreator = mock(RequestManager.class);
-        var request = mock(InferenceRequest.class);
-        when(request.getRequestCreator()).thenReturn(requestCreator);
-
-        new SingleRequestManager(mock(RetryingHttpSender.class)).execute(mock(InferenceRequest.class), HttpClientContext.create());
-        verifyNoInteractions(requestCreator);
-    }
-}
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java
index ee3403492c423..974b31e73b499 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/SenderServiceTests.java
@@ -33,7 +33,6 @@
 
 import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool;
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
-import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -59,7 +58,7 @@ public void testStart_InitializesTheSender() throws IOException {
         var sender = mock(Sender.class);
 
         var factory = mock(HttpRequestSender.Factory.class);
-        when(factory.createSender(anyString())).thenReturn(sender);
+        when(factory.createSender()).thenReturn(sender);
 
         try (var service = new TestSenderService(factory, createWithEmptySettings(threadPool))) {
             PlainActionFuture<Boolean> listener = new PlainActionFuture<>();
@@ -67,7 +66,7 @@ public void testStart_InitializesTheSender() throws IOException {
 
             listener.actionGet(TIMEOUT);
             verify(sender, times(1)).start();
-            verify(factory, times(1)).createSender(anyString());
+            verify(factory, times(1)).createSender();
         }
 
         verify(sender, times(1)).close();
@@ -79,7 +78,7 @@ public void testStart_CallingStartTwiceKeepsSameSenderReference() throws IOExcep
         var sender = mock(Sender.class);
 
         var factory = mock(HttpRequestSender.Factory.class);
-        when(factory.createSender(anyString())).thenReturn(sender);
+        when(factory.createSender()).thenReturn(sender);
 
         try (var service = new TestSenderService(factory, createWithEmptySettings(threadPool))) {
             PlainActionFuture<Boolean> listener = new PlainActionFuture<>();
@@ -89,7 +88,7 @@ public void testStart_CallingStartTwiceKeepsSameSenderReference() throws IOExcep
             service.start(mock(Model.class), listener);
             listener.actionGet(TIMEOUT);
 
-            verify(factory, times(1)).createSender(anyString());
+            verify(factory, times(1)).createSender();
             verify(sender, times(2)).start();
         }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java
index 5869366ac2e22..cacbba82446f1 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java
@@ -76,7 +76,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -819,7 +818,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAzureAiStudioModel() throws IOExc
         var sender = mock(Sender.class);
 
         var factory = mock(HttpRequestSender.Factory.class);
-        when(factory.createSender(anyString())).thenReturn(sender);
+        when(factory.createSender()).thenReturn(sender);
 
         var mockModel = getInvalidModel("model_id", "service_name");
 
@@ -841,7 +840,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAzureAiStudioModel() throws IOExc
                 is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.")
             );
 
-            verify(factory, times(1)).createSender(anyString());
+            verify(factory, times(1)).createSender();
             verify(sender, times(1)).start();
         }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java
index 79d6e384d7693..d46a5f190017a 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java
@@ -112,7 +112,8 @@ public void testToFilteredXContent_WritesAllValues() throws IOException {
         String xContentResult = Strings.toString(builder);
 
         assertThat(xContentResult, CoreMatchers.is("""
-            {"target":"target_value","provider":"openai","endpoint_type":"token"}"""));
+            {"target":"target_value","provider":"openai","endpoint_type":"token",""" + """
+            "rate_limit":{"requests_per_minute":3}}"""));
     }
 
     public static HashMap<String, Object> createRequestSettingsMap(String target, String provider, String endpointType) {
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsServiceSettingsTests.java
index 283bfa1490df2..a592dd6e1f956 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/embeddings/AzureAiStudioEmbeddingsServiceSettingsTests.java
@@ -295,7 +295,7 @@ public void testToFilteredXContent_WritesAllValues_ExceptDimensionsSetByUser() t
 
         assertThat(xContentResult, CoreMatchers.is("""
             {"target":"target_value","provider":"openai","endpoint_type":"token",""" + """
-            "dimensions":1024,"max_input_tokens":512}"""));
+            "rate_limit":{"requests_per_minute":3},"dimensions":1024,"max_input_tokens":512}"""));
     }
 
     public static HashMap<String, Object> createRequestSettingsMap(
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java
index 9fe8b472b22a5..bb3407056d573 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java
@@ -73,7 +73,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -594,7 +593,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAzureOpenAiModel() throws IOExcep
         var sender = mock(Sender.class);
 
         var factory = mock(HttpRequestSender.Factory.class);
-        when(factory.createSender(anyString())).thenReturn(sender);
+        when(factory.createSender()).thenReturn(sender);
 
         var mockModel = getInvalidModel("model_id", "service_name");
 
@@ -616,7 +615,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAzureOpenAiModel() throws IOExcep
                 is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.")
             );
 
-            verify(factory, times(1)).createSender(anyString());
+            verify(factory, times(1)).createSender();
             verify(sender, times(1)).start();
         }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettingsTests.java
index 46e514c8b16c4..797cad8f300ae 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/completion/AzureOpenAiCompletionServiceSettingsTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiServiceFields;
 
 import java.io.IOException;
@@ -46,7 +47,8 @@ public void testFromMap_Request_CreatesSettingsCorrectly() {
                     AzureOpenAiServiceFields.API_VERSION,
                     apiVersion
                 )
-            )
+            ),
+            ConfigurationParseContext.PERSISTENT
         );
 
         assertThat(serviceSettings, is(new AzureOpenAiCompletionServiceSettings(resourceName, deploymentId, apiVersion, null)));
@@ -63,18 +65,6 @@ public void testToXContent_WritesAllValues() throws IOException {
             {"resource_name":"resource","deployment_id":"deployment","api_version":"2024","rate_limit":{"requests_per_minute":120}}"""));
     }
 
-    public void testToFilteredXContent_WritesAllValues_Except_RateLimit() throws IOException {
-        var entity = new AzureOpenAiCompletionServiceSettings("resource", "deployment", "2024", null);
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        var filteredXContent = entity.getFilteredXContentObject();
-        filteredXContent.toXContent(builder, null);
-        String xContentResult = Strings.toString(builder);
-
-        assertThat(xContentResult, is("""
-            {"resource_name":"resource","deployment_id":"deployment","api_version":"2024"}"""));
-    }
-
     @Override
     protected Writeable.Reader<AzureOpenAiCompletionServiceSettings> instanceReader() {
         return AzureOpenAiCompletionServiceSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettingsTests.java
index f4c6f9b2a4f07..cbb9eea223802 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettingsTests.java
@@ -389,7 +389,7 @@ public void testToXContent_WritesAllValues() throws IOException {
             "dimensions":1024,"max_input_tokens":512,"rate_limit":{"requests_per_minute":3},"dimensions_set_by_user":false}"""));
     }
 
-    public void testToFilteredXContent_WritesAllValues_Except_DimensionsSetByUser_RateLimit() throws IOException {
+    public void testToFilteredXContent_WritesAllValues_Except_DimensionsSetByUser() throws IOException {
         var entity = new AzureOpenAiEmbeddingsServiceSettings(
             "resource",
             "deployment",
@@ -408,7 +408,7 @@ public void testToFilteredXContent_WritesAllValues_Except_DimensionsSetByUser_Ra
 
         assertThat(xContentResult, is("""
             {"resource_name":"resource","deployment_id":"deployment","api_version":"apiVersion",""" + """
-            "dimensions":1024,"max_input_tokens":512}"""));
+            "dimensions":1024,"max_input_tokens":512,"rate_limit":{"requests_per_minute":1}}"""));
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
index f06fee4b0b9c4..902d96be29738 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java
@@ -73,7 +73,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -613,7 +612,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotCohereModel() throws IOException
         var sender = mock(Sender.class);
 
         var factory = mock(HttpRequestSender.Factory.class);
-        when(factory.createSender(anyString())).thenReturn(sender);
+        when(factory.createSender()).thenReturn(sender);
 
         var mockModel = getInvalidModel("model_id", "service_name");
 
@@ -635,7 +634,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotCohereModel() throws IOException
                 is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.")
             );
 
-            verify(factory, times(1)).createSender(anyString());
+            verify(factory, times(1)).createSender();
             verify(sender, times(1)).start();
         }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModelTests.java
index aac04e301ece7..b9fc7ee7b9952 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModelTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionModelTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.inference.EmptyTaskSettings;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 import java.util.HashMap;
@@ -28,7 +29,8 @@ public void testCreateModel_AlwaysWithEmptyTaskSettings() {
             "service",
             new HashMap<>(Map.of()),
             new HashMap<>(Map.of("model", "overridden model")),
-            null
+            null,
+            ConfigurationParseContext.PERSISTENT
         );
 
         assertThat(model.getTaskSettings(), is(EmptyTaskSettings.INSTANCE));
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettingsTests.java
index f4cab3c2b0f1e..ed8bc90d32140 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettingsTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.ServiceFields;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests;
@@ -34,7 +35,8 @@ public void testFromMap_WithRateLimitSettingsNull() {
         var model = "model";
 
         var serviceSettings = CohereCompletionServiceSettings.fromMap(
-            new HashMap<>(Map.of(ServiceFields.URL, url, ServiceFields.MODEL_ID, model))
+            new HashMap<>(Map.of(ServiceFields.URL, url, ServiceFields.MODEL_ID, model)),
+            ConfigurationParseContext.PERSISTENT
         );
 
         assertThat(serviceSettings, is(new CohereCompletionServiceSettings(url, model, null)));
@@ -55,7 +57,8 @@ public void testFromMap_WithRateLimitSettings() {
                     RateLimitSettings.FIELD_NAME,
                     new HashMap<>(Map.of(RateLimitSettings.REQUESTS_PER_MINUTE_FIELD, requestsPerMinute))
                 )
-            )
+            ),
+            ConfigurationParseContext.PERSISTENT
         );
 
         assertThat(serviceSettings, is(new CohereCompletionServiceSettings(url, model, new RateLimitSettings(requestsPerMinute))));
@@ -72,18 +75,6 @@ public void testToXContent_WritesAllValues() throws IOException {
             {"url":"url","model_id":"model","rate_limit":{"requests_per_minute":3}}"""));
     }
 
-    public void testToXContent_WithFilteredObject_WritesAllValues_Except_RateLimit() throws IOException {
-        var serviceSettings = new CohereCompletionServiceSettings("url", "model", new RateLimitSettings(3));
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        var filteredXContent = serviceSettings.getFilteredXContentObject();
-        filteredXContent.toXContent(builder, null);
-        String xContentResult = Strings.toString(builder);
-
-        assertThat(xContentResult, is("""
-            {"url":"url","model_id":"model"}"""));
-    }
-
     @Override
     protected Writeable.Reader<CohereCompletionServiceSettings> instanceReader() {
         return CohereCompletionServiceSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettingsTests.java
index 6f8fe6344b57f..73ebd6c6c0505 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsServiceSettingsTests.java
@@ -331,21 +331,6 @@ public void testToXContent_WritesAllValues() throws IOException {
             "rate_limit":{"requests_per_minute":3},"embedding_type":"byte"}"""));
     }
 
-    public void testToXContent_WritesAllValues_Except_RateLimit() throws IOException {
-        var serviceSettings = new CohereEmbeddingsServiceSettings(
-            new CohereServiceSettings("url", SimilarityMeasure.COSINE, 5, 10, "model_id", new RateLimitSettings(3)),
-            CohereEmbeddingType.INT8
-        );
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        var filteredXContent = serviceSettings.getFilteredXContentObject();
-        filteredXContent.toXContent(builder, null);
-        String xContentResult = Strings.toString(builder);
-        assertThat(xContentResult, is("""
-            {"url":"url","similarity":"cosine","dimensions":5,"max_input_tokens":10,"model_id":"model_id",""" + """
-            "embedding_type":"byte"}"""));
-    }
-
     @Override
     protected Writeable.Reader<CohereEmbeddingsServiceSettings> instanceReader() {
         return CohereEmbeddingsServiceSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java
index 4943ddf74fda1..1ce5a9fb12833 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java
@@ -51,20 +51,6 @@ public void testToXContent_WritesAllValues() throws IOException {
             "rate_limit":{"requests_per_minute":3}}"""));
     }
 
-    public void testToXContent_WritesAllValues_Except_RateLimit() throws IOException {
-        var serviceSettings = new CohereRerankServiceSettings(
-            new CohereServiceSettings("url", SimilarityMeasure.COSINE, 5, 10, "model_id", new RateLimitSettings(3))
-        );
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        var filteredXContent = serviceSettings.getFilteredXContentObject();
-        filteredXContent.toXContent(builder, null);
-        String xContentResult = Strings.toString(builder);
-        // TODO we probably shouldn't allow configuring these fields for reranking
-        assertThat(xContentResult, is("""
-            {"url":"url","similarity":"cosine","dimensions":5,"max_input_tokens":10,"model_id":"model_id"}"""));
-    }
-
     @Override
     protected Writeable.Reader<CohereRerankServiceSettings> instanceReader() {
         return CohereRerankServiceSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java
index 32e912ff8529a..110276e63d077 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java
@@ -73,7 +73,6 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.hasSize;
-import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -494,7 +493,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotGoogleAiStudioModel() throws IOEx
         var sender = mock(Sender.class);
 
         var factory = mock(HttpRequestSender.Factory.class);
-        when(factory.createSender(anyString())).thenReturn(sender);
+        when(factory.createSender()).thenReturn(sender);
 
         var mockModel = getInvalidModel("model_id", "service_name");
 
@@ -516,7 +515,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotGoogleAiStudioModel() throws IOEx
                 is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.")
             );
 
-            verify(factory, times(1)).createSender(anyString());
+            verify(factory, times(1)).createSender();
             verify(sender, times(1)).start();
         }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java
index 025317fbe025a..f4c13db78c4bc 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionModelTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.inference.EmptyTaskSettings;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings;
 
 import java.net.URISyntaxException;
@@ -28,7 +29,8 @@ public void testCreateModel_AlwaysWithEmptyTaskSettings() {
             "service",
             new HashMap<>(Map.of("model_id", "model")),
             new HashMap<>(Map.of()),
-            null
+            null,
+            ConfigurationParseContext.PERSISTENT
         );
 
         assertThat(model.getTaskSettings(), is(EmptyTaskSettings.INSTANCE));
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettingsTests.java
index 46e6e60af493c..6652af26e09e1 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/completion/GoogleAiStudioCompletionServiceSettingsTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.ServiceFields;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests;
 
@@ -31,7 +32,10 @@ public static GoogleAiStudioCompletionServiceSettings createRandom() {
     public void testFromMap_Request_CreatesSettingsCorrectly() {
         var model = "some model";
 
-        var serviceSettings = GoogleAiStudioCompletionServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.MODEL_ID, model)));
+        var serviceSettings = GoogleAiStudioCompletionServiceSettings.fromMap(
+            new HashMap<>(Map.of(ServiceFields.MODEL_ID, model)),
+            ConfigurationParseContext.PERSISTENT
+        );
 
         assertThat(serviceSettings, is(new GoogleAiStudioCompletionServiceSettings(model, null)));
     }
@@ -47,18 +51,6 @@ public void testToXContent_WritesAllValues() throws IOException {
             {"model_id":"model","rate_limit":{"requests_per_minute":360}}"""));
     }
 
-    public void testToFilteredXContent_WritesAllValues_Except_RateLimit() throws IOException {
-        var entity = new GoogleAiStudioCompletionServiceSettings("model", null);
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        var filteredXContent = entity.getFilteredXContentObject();
-        filteredXContent.toXContent(builder, null);
-        String xContentResult = Strings.toString(builder);
-
-        assertThat(xContentResult, is("""
-            {"model_id":"model"}"""));
-    }
-
     @Override
     protected Writeable.Reader<GoogleAiStudioCompletionServiceSettings> instanceReader() {
         return GoogleAiStudioCompletionServiceSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettingsTests.java
index b5fbd28b476ba..cc195333adfd4 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/embeddings/GoogleAiStudioEmbeddingsServiceSettingsTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.ServiceFields;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests;
 
@@ -55,7 +56,8 @@ public void testFromMap_Request_CreatesSettingsCorrectly() {
                     ServiceFields.SIMILARITY,
                     similarity.toString()
                 )
-            )
+            ),
+            ConfigurationParseContext.PERSISTENT
         );
 
         assertThat(serviceSettings, is(new GoogleAiStudioEmbeddingsServiceSettings(model, maxInputTokens, dims, similarity, null)));
@@ -80,23 +82,6 @@ public void testToXContent_WritesAllValues() throws IOException {
             }"""));
     }
 
-    public void testToFilteredXContent_WritesAllValues_Except_RateLimit() throws IOException {
-        var entity = new GoogleAiStudioEmbeddingsServiceSettings("model", 1024, 8, SimilarityMeasure.DOT_PRODUCT, null);
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        var filteredXContent = entity.getFilteredXContentObject();
-        filteredXContent.toXContent(builder, null);
-        String xContentResult = Strings.toString(builder);
-
-        assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString("""
-            {
-                "model_id":"model",
-                "max_input_tokens": 1024,
-                "dimensions": 8,
-                "similarity": "dot_product"
-            }"""));
-    }
-
     @Override
     protected Writeable.Reader<GoogleAiStudioEmbeddingsServiceSettings> instanceReader() {
         return GoogleAiStudioEmbeddingsServiceSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java
index 398b21312a03a..fd7e1b48b7e03 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender;
 import org.elasticsearch.xpack.inference.external.http.sender.Sender;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.ServiceComponents;
 import org.junit.After;
 import org.junit.Before;
@@ -33,7 +34,6 @@
 import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool;
 import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings;
 import static org.hamcrest.CoreMatchers.is;
-import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -59,7 +59,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotHuggingFaceModel() throws IOExcep
         var sender = mock(Sender.class);
 
         var factory = mock(HttpRequestSender.Factory.class);
-        when(factory.createSender(anyString())).thenReturn(sender);
+        when(factory.createSender()).thenReturn(sender);
 
         var mockModel = getInvalidModel("model_id", "service_name");
 
@@ -81,7 +81,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotHuggingFaceModel() throws IOExcep
                 is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.")
             );
 
-            verify(factory, times(1)).createSender(anyString());
+            verify(factory, times(1)).createSender();
             verify(sender, times(1)).start();
         }
 
@@ -111,7 +111,8 @@ protected HuggingFaceModel createModel(
             TaskType taskType,
             Map<String, Object> serviceSettings,
             Map<String, Object> secretSettings,
-            String failureMessage
+            String failureMessage,
+            ConfigurationParseContext context
         ) {
             return null;
         }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettingsTests.java
index 91b91593adee7..04e9697b08877 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettingsTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.ServiceFields;
 import org.elasticsearch.xpack.inference.services.ServiceUtils;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
@@ -57,7 +58,10 @@ public void testFromMap() {
         var dims = 384;
         var maxInputTokens = 128;
         {
-            var serviceSettings = HuggingFaceServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, url)));
+            var serviceSettings = HuggingFaceServiceSettings.fromMap(
+                new HashMap<>(Map.of(ServiceFields.URL, url)),
+                ConfigurationParseContext.PERSISTENT
+            );
             assertThat(serviceSettings, is(new HuggingFaceServiceSettings(url)));
         }
         {
@@ -73,7 +77,8 @@ public void testFromMap() {
                         ServiceFields.MAX_INPUT_TOKENS,
                         maxInputTokens
                     )
-                )
+                ),
+                ConfigurationParseContext.PERSISTENT
             );
             assertThat(
                 serviceSettings,
@@ -95,7 +100,8 @@ public void testFromMap() {
                         RateLimitSettings.FIELD_NAME,
                         new HashMap<>(Map.of(RateLimitSettings.REQUESTS_PER_MINUTE_FIELD, 3))
                     )
-                )
+                ),
+                ConfigurationParseContext.PERSISTENT
             );
             assertThat(
                 serviceSettings,
@@ -105,7 +111,10 @@ public void testFromMap() {
     }
 
     public void testFromMap_MissingUrl_ThrowsError() {
-        var thrownException = expectThrows(ValidationException.class, () -> HuggingFaceServiceSettings.fromMap(new HashMap<>()));
+        var thrownException = expectThrows(
+            ValidationException.class,
+            () -> HuggingFaceServiceSettings.fromMap(new HashMap<>(), ConfigurationParseContext.PERSISTENT)
+        );
 
         assertThat(
             thrownException.getMessage(),
@@ -118,7 +127,7 @@ public void testFromMap_MissingUrl_ThrowsError() {
     public void testFromMap_EmptyUrl_ThrowsError() {
         var thrownException = expectThrows(
             ValidationException.class,
-            () -> HuggingFaceServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, "")))
+            () -> HuggingFaceServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, "")), ConfigurationParseContext.PERSISTENT)
         );
 
         assertThat(
@@ -136,7 +145,7 @@ public void testFromMap_InvalidUrl_ThrowsError() {
         var url = "https://www.abc^.com";
         var thrownException = expectThrows(
             ValidationException.class,
-            () -> HuggingFaceServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, url)))
+            () -> HuggingFaceServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, url)), ConfigurationParseContext.PERSISTENT)
         );
 
         assertThat(
@@ -152,7 +161,10 @@ public void testFromMap_InvalidSimilarity_ThrowsError() {
         var similarity = "by_size";
         var thrownException = expectThrows(
             ValidationException.class,
-            () -> HuggingFaceServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, url, ServiceFields.SIMILARITY, similarity)))
+            () -> HuggingFaceServiceSettings.fromMap(
+                new HashMap<>(Map.of(ServiceFields.URL, url, ServiceFields.SIMILARITY, similarity)),
+                ConfigurationParseContext.PERSISTENT
+            )
         );
 
         assertThat(
@@ -175,18 +187,6 @@ public void testToXContent_WritesAllValues() throws IOException {
             {"url":"url","rate_limit":{"requests_per_minute":3}}"""));
     }
 
-    public void testToXContent_WritesAllValues_Except_RateLimit() throws IOException {
-        var serviceSettings = new HuggingFaceServiceSettings(ServiceUtils.createUri("url"), null, null, null, new RateLimitSettings(3));
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        var filteredXContent = serviceSettings.getFilteredXContentObject();
-        filteredXContent.toXContent(builder, null);
-        String xContentResult = org.elasticsearch.common.Strings.toString(builder);
-
-        assertThat(xContentResult, is("""
-            {"url":"url"}"""));
-    }
-
     @Override
     protected Writeable.Reader<HuggingFaceServiceSettings> instanceReader() {
         return HuggingFaceServiceSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettingsTests.java
index 57f9c59b65e12..2a44429687fb3 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettingsTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.ServiceUtils;
 import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings;
 
@@ -32,7 +33,10 @@ public static HuggingFaceElserServiceSettings createRandom() {
 
     public void testFromMap() {
         var url = "https://www.abc.com";
-        var serviceSettings = HuggingFaceElserServiceSettings.fromMap(new HashMap<>(Map.of(HuggingFaceElserServiceSettings.URL, url)));
+        var serviceSettings = HuggingFaceElserServiceSettings.fromMap(
+            new HashMap<>(Map.of(HuggingFaceElserServiceSettings.URL, url)),
+            ConfigurationParseContext.PERSISTENT
+        );
 
         assertThat(new HuggingFaceElserServiceSettings(url), is(serviceSettings));
     }
@@ -40,7 +44,10 @@ public void testFromMap() {
     public void testFromMap_EmptyUrl_ThrowsError() {
         var thrownException = expectThrows(
             ValidationException.class,
-            () -> HuggingFaceElserServiceSettings.fromMap(new HashMap<>(Map.of(HuggingFaceElserServiceSettings.URL, "")))
+            () -> HuggingFaceElserServiceSettings.fromMap(
+                new HashMap<>(Map.of(HuggingFaceElserServiceSettings.URL, "")),
+                ConfigurationParseContext.PERSISTENT
+            )
         );
 
         assertThat(
@@ -55,7 +62,10 @@ public void testFromMap_EmptyUrl_ThrowsError() {
     }
 
     public void testFromMap_MissingUrl_ThrowsError() {
-        var thrownException = expectThrows(ValidationException.class, () -> HuggingFaceElserServiceSettings.fromMap(new HashMap<>()));
+        var thrownException = expectThrows(
+            ValidationException.class,
+            () -> HuggingFaceElserServiceSettings.fromMap(new HashMap<>(), ConfigurationParseContext.PERSISTENT)
+        );
 
         assertThat(
             thrownException.getMessage(),
@@ -72,7 +82,10 @@ public void testFromMap_InvalidUrl_ThrowsError() {
         var url = "https://www.abc^.com";
         var thrownException = expectThrows(
             ValidationException.class,
-            () -> HuggingFaceElserServiceSettings.fromMap(new HashMap<>(Map.of(HuggingFaceElserServiceSettings.URL, url)))
+            () -> HuggingFaceElserServiceSettings.fromMap(
+                new HashMap<>(Map.of(HuggingFaceElserServiceSettings.URL, url)),
+                ConfigurationParseContext.PERSISTENT
+            )
         );
 
         assertThat(
@@ -98,18 +111,6 @@ public void testToXContent_WritesAllValues() throws IOException {
             {"url":"url","max_input_tokens":512,"rate_limit":{"requests_per_minute":3}}"""));
     }
 
-    public void testToXContent_WritesAllValues_Except_RateLimit() throws IOException {
-        var serviceSettings = new HuggingFaceElserServiceSettings(ServiceUtils.createUri("url"), new RateLimitSettings(3));
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        var filteredXContent = serviceSettings.getFilteredXContentObject();
-        filteredXContent.toXContent(builder, null);
-        String xContentResult = org.elasticsearch.common.Strings.toString(builder);
-
-        assertThat(xContentResult, is("""
-            {"url":"url","max_input_tokens":512}"""));
-    }
-
     @Override
     protected Writeable.Reader<HuggingFaceElserServiceSettings> instanceReader() {
         return HuggingFaceElserServiceSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java
index 3ead273e78110..624b24e611340 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java
@@ -67,7 +67,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -393,7 +392,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotMistralEmbeddingsModel() throws I
         var sender = mock(Sender.class);
 
         var factory = mock(HttpRequestSender.Factory.class);
-        when(factory.createSender(anyString())).thenReturn(sender);
+        when(factory.createSender()).thenReturn(sender);
 
         var mockModel = getInvalidModel("model_id", "service_name");
 
@@ -415,7 +414,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotMistralEmbeddingsModel() throws I
                 is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.")
             );
 
-            verify(factory, times(1)).createSender(anyString());
+            verify(factory, times(1)).createSender();
             verify(sender, times(1)).start();
         }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettingsTests.java
index 13f43a5f31ad3..076986acdcee6 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsServiceSettingsTests.java
@@ -98,18 +98,6 @@ public void testToXContent_WritesAllValues() throws IOException {
             "rate_limit":{"requests_per_minute":3}}"""));
     }
 
-    public void testToFilteredXContent_WritesFilteredValues() throws IOException {
-        var entity = new MistralEmbeddingsServiceSettings("model_name", 1024, 512, null, new RateLimitSettings(3));
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        var filteredXContent = entity.getFilteredXContentObject();
-        filteredXContent.toXContent(builder, null);
-        String xContentResult = Strings.toString(builder);
-
-        assertThat(xContentResult, CoreMatchers.is("""
-            {"model":"model_name","dimensions":1024,"max_input_tokens":512}"""));
-    }
-
     public void testStreamInputAndOutput_WritesValuesCorrectly() throws IOException {
         var outputBuffer = new BytesStreamOutput();
         var settings = new MistralEmbeddingsServiceSettings("model_name", 1024, 512, null, new RateLimitSettings(3));
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
index cbac29c452772..41995235565df 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java
@@ -72,7 +72,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -675,7 +674,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotOpenAiModel() throws IOException
         var sender = mock(Sender.class);
 
         var factory = mock(HttpRequestSender.Factory.class);
-        when(factory.createSender(anyString())).thenReturn(sender);
+        when(factory.createSender()).thenReturn(sender);
 
         var mockModel = getInvalidModel("model_id", "service_name");
 
@@ -697,7 +696,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotOpenAiModel() throws IOException
                 is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.")
             );
 
-            verify(factory, times(1)).createSender(anyString());
+            verify(factory, times(1)).createSender();
             verify(sender, times(1)).start();
         }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettingsTests.java
index 186ca89426418..051a9bc6d9bef 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettingsTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 import org.elasticsearch.xpack.inference.services.ServiceFields;
 import org.elasticsearch.xpack.inference.services.ServiceUtils;
 import org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields;
@@ -48,7 +49,8 @@ public void testFromMap_Request_CreatesSettingsCorrectly() {
                     ServiceFields.MAX_INPUT_TOKENS,
                     maxInputTokens
                 )
-            )
+            ),
+            ConfigurationParseContext.PERSISTENT
         );
 
         assertThat(
@@ -77,7 +79,8 @@ public void testFromMap_Request_CreatesSettingsCorrectly_WithRateLimit() {
                     RateLimitSettings.FIELD_NAME,
                     new HashMap<>(Map.of(RateLimitSettings.REQUESTS_PER_MINUTE_FIELD, rateLimit))
                 )
-            )
+            ),
+            ConfigurationParseContext.PERSISTENT
         );
 
         assertThat(
@@ -101,7 +104,8 @@ public void testFromMap_MissingUrl_DoesNotThrowException() {
                     ServiceFields.MAX_INPUT_TOKENS,
                     maxInputTokens
                 )
-            )
+            ),
+            ConfigurationParseContext.PERSISTENT
         );
 
         assertNull(serviceSettings.uri());
@@ -113,7 +117,10 @@ public void testFromMap_MissingUrl_DoesNotThrowException() {
     public void testFromMap_EmptyUrl_ThrowsError() {
         var thrownException = expectThrows(
             ValidationException.class,
-            () -> OpenAiChatCompletionServiceSettings.fromMap(new HashMap<>(Map.of(ServiceFields.URL, "", ServiceFields.MODEL_ID, "model")))
+            () -> OpenAiChatCompletionServiceSettings.fromMap(
+                new HashMap<>(Map.of(ServiceFields.URL, "", ServiceFields.MODEL_ID, "model")),
+                ConfigurationParseContext.PERSISTENT
+            )
         );
 
         assertThat(
@@ -132,7 +139,8 @@ public void testFromMap_MissingOrganization_DoesNotThrowException() {
         var maxInputTokens = 8192;
 
         var serviceSettings = OpenAiChatCompletionServiceSettings.fromMap(
-            new HashMap<>(Map.of(ServiceFields.MODEL_ID, modelId, ServiceFields.MAX_INPUT_TOKENS, maxInputTokens))
+            new HashMap<>(Map.of(ServiceFields.MODEL_ID, modelId, ServiceFields.MAX_INPUT_TOKENS, maxInputTokens)),
+            ConfigurationParseContext.PERSISTENT
         );
 
         assertNull(serviceSettings.uri());
@@ -144,7 +152,8 @@ public void testFromMap_EmptyOrganization_ThrowsError() {
         var thrownException = expectThrows(
             ValidationException.class,
             () -> OpenAiChatCompletionServiceSettings.fromMap(
-                new HashMap<>(Map.of(OpenAiServiceFields.ORGANIZATION, "", ServiceFields.MODEL_ID, "model"))
+                new HashMap<>(Map.of(OpenAiServiceFields.ORGANIZATION, "", ServiceFields.MODEL_ID, "model")),
+                ConfigurationParseContext.PERSISTENT
             )
         );
 
@@ -164,7 +173,8 @@ public void testFromMap_InvalidUrl_ThrowsError() {
         var thrownException = expectThrows(
             ValidationException.class,
             () -> OpenAiChatCompletionServiceSettings.fromMap(
-                new HashMap<>(Map.of(ServiceFields.URL, url, ServiceFields.MODEL_ID, "model"))
+                new HashMap<>(Map.of(ServiceFields.URL, url, ServiceFields.MODEL_ID, "model")),
+                ConfigurationParseContext.PERSISTENT
             )
         );
 
@@ -213,19 +223,6 @@ public void testToXContent_DoesNotWriteOptionalValues() throws IOException {
             {"model_id":"model","rate_limit":{"requests_per_minute":500}}"""));
     }
 
-    public void testToXContent_WritesAllValues_Except_RateLimit() throws IOException {
-        var serviceSettings = new OpenAiChatCompletionServiceSettings("model", "url", "org", 1024, new RateLimitSettings(2));
-
-        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        var filteredXContent = serviceSettings.getFilteredXContentObject();
-        filteredXContent.toXContent(builder, null);
-        String xContentResult = org.elasticsearch.common.Strings.toString(builder);
-
-        assertThat(xContentResult, is("""
-            {"model_id":"model","url":"url","organization_id":"org",""" + """
-            "max_input_tokens":1024}"""));
-    }
-
     @Override
     protected Writeable.Reader<OpenAiChatCompletionServiceSettings> instanceReader() {
         return OpenAiChatCompletionServiceSettings::new;
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java
index 438f895fe48ad..cc0004a2d678c 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettingsTests.java
@@ -406,7 +406,7 @@ public void testToFilteredXContent_WritesAllValues_ExceptDimensionsSetByUser() t
 
         assertThat(xContentResult, is("""
             {"model_id":"model","url":"url","organization_id":"org","similarity":"dot_product",""" + """
-            "dimensions":1,"max_input_tokens":2}"""));
+            "dimensions":1,"max_input_tokens":2,"rate_limit":{"requests_per_minute":3000}}"""));
     }
 
     public void testToFilteredXContent_WritesAllValues_WithSpecifiedRateLimit() throws IOException {
@@ -428,7 +428,7 @@ public void testToFilteredXContent_WritesAllValues_WithSpecifiedRateLimit() thro
 
         assertThat(xContentResult, is("""
             {"model_id":"model","url":"url","organization_id":"org","similarity":"dot_product",""" + """
-            "dimensions":1,"max_input_tokens":2}"""));
+            "dimensions":1,"max_input_tokens":2,"rate_limit":{"requests_per_minute":2000}}"""));
     }
 
     @Override
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettingsTests.java
index cdee7c452ff52..7e3bdd6b8e5dc 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettingsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/settings/RateLimitSettingsTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.inference.services.settings;
 
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -14,6 +15,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.inference.services.ConfigurationParseContext;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -49,7 +51,7 @@ public void testOf() {
         Map<String, Object> settings = new HashMap<>(
             Map.of(RateLimitSettings.FIELD_NAME, new HashMap<>(Map.of(RateLimitSettings.REQUESTS_PER_MINUTE_FIELD, 100)))
         );
-        var res = RateLimitSettings.of(settings, new RateLimitSettings(1), validation);
+        var res = RateLimitSettings.of(settings, new RateLimitSettings(1), validation, "test", ConfigurationParseContext.PERSISTENT);
 
         assertThat(res, is(new RateLimitSettings(100)));
         assertTrue(validation.validationErrors().isEmpty());
@@ -60,7 +62,7 @@ public void testOf_UsesDefaultValue_WhenRateLimit_IsAbsent() {
         Map<String, Object> settings = new HashMap<>(
             Map.of("abc", new HashMap<>(Map.of(RateLimitSettings.REQUESTS_PER_MINUTE_FIELD, 100)))
         );
-        var res = RateLimitSettings.of(settings, new RateLimitSettings(1), validation);
+        var res = RateLimitSettings.of(settings, new RateLimitSettings(1), validation, "test", ConfigurationParseContext.PERSISTENT);
 
         assertThat(res, is(new RateLimitSettings(1)));
         assertTrue(validation.validationErrors().isEmpty());
@@ -69,12 +71,24 @@ public void testOf_UsesDefaultValue_WhenRateLimit_IsAbsent() {
     public void testOf_UsesDefaultValue_WhenRequestsPerMinute_IsAbsent() {
         var validation = new ValidationException();
         Map<String, Object> settings = new HashMap<>(Map.of(RateLimitSettings.FIELD_NAME, new HashMap<>(Map.of("abc", 100))));
-        var res = RateLimitSettings.of(settings, new RateLimitSettings(1), validation);
+        var res = RateLimitSettings.of(settings, new RateLimitSettings(1), validation, "test", ConfigurationParseContext.PERSISTENT);
 
         assertThat(res, is(new RateLimitSettings(1)));
         assertTrue(validation.validationErrors().isEmpty());
     }
 
+    public void testOf_ThrowsException_WithUnknownField_InRequestContext() {
+        var validation = new ValidationException();
+        Map<String, Object> settings = new HashMap<>(Map.of(RateLimitSettings.FIELD_NAME, new HashMap<>(Map.of("abc", 100))));
+
+        var exception = expectThrows(
+            ElasticsearchStatusException.class,
+            () -> RateLimitSettings.of(settings, new RateLimitSettings(1), validation, "test", ConfigurationParseContext.REQUEST)
+        );
+
+        assertThat(exception.getMessage(), is("Model configuration contains settings [{abc=100}] unknown to the [test] service"));
+    }
+
     public void testToXContent() throws IOException {
         var settings = new RateLimitSettings(100);
 

From dc13b75656146b59d5e52be7fd78b288237cf7bd Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Wed, 5 Jun 2024 15:35:31 +0200
Subject: [PATCH 167/208] [Inference API] Add `text_embedding` task type to
 Google AI Studio docs (#109307)

---
 docs/reference/inference/put-inference.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc
index f805bc0cc92f7..e7d66e930e81f 100644
--- a/docs/reference/inference/put-inference.asciidoc
+++ b/docs/reference/inference/put-inference.asciidoc
@@ -74,7 +74,7 @@ Available services:
 * `cohere`: specify the `completion`, `text_embedding` or the `rerank` task type to use the Cohere service.
 * `elasticsearch`: specify the `text_embedding` task type to use the E5 built-in model or text embedding models uploaded by Eland.
 * `elser`: specify the `sparse_embedding` task type to use the ELSER service.
-* `googleaistudio`: specify the `completion` task to use the Google AI Studio service.
+* `googleaistudio`: specify the `completion` or `text_embeddig` task to use the Google AI Studio service.
 * `hugging_face`: specify the `text_embedding` task type to use the Hugging Face service.
 * `openai`: specify the `completion` or `text_embedding` task type to use the OpenAI service.
 

From 19eabb928dffb232b914926f05cb8eac14a88997 Mon Sep 17 00:00:00 2001
From: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co>
Date: Wed, 5 Jun 2024 15:07:01 +0000
Subject: [PATCH 168/208] Bump versions after 8.14.0 release

---
 .buildkite/pipelines/intake.yml                  |  2 +-
 .buildkite/pipelines/periodic-packaging.yml      | 12 ++++++------
 .buildkite/pipelines/periodic.yml                | 16 ++++++++--------
 .ci/bwcVersions                                  |  4 ++--
 .ci/snapshotBwcVersions                          |  3 +--
 .../src/main/java/org/elasticsearch/Version.java |  2 +-
 .../org/elasticsearch/TransportVersions.csv      |  1 +
 .../org/elasticsearch/index/IndexVersions.csv    |  1 +
 8 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
index 944230377d077..c5b079c39fbc1 100644
--- a/.buildkite/pipelines/intake.yml
+++ b/.buildkite/pipelines/intake.yml
@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.22", "8.13.5", "8.14.0", "8.15.0"]
+            BWC_VERSION: ["7.17.22", "8.14.1", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
index 5ac361c810627..378a7c5c9c5d2 100644
--- a/.buildkite/pipelines/periodic-packaging.yml
+++ b/.buildkite/pipelines/periodic-packaging.yml
@@ -529,8 +529,8 @@ steps:
         env:
           BWC_VERSION: 8.12.2
 
-      - label: "{{matrix.image}} / 8.13.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.5
+      - label: "{{matrix.image}} / 8.13.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -543,10 +543,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.5
+          BWC_VERSION: 8.13.4
 
-      - label: "{{matrix.image}} / 8.14.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.0
+      - label: "{{matrix.image}} / 8.14.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -559,7 +559,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.14.0
+          BWC_VERSION: 8.14.1
 
       - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0
diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
index 7ba46f0f0951c..1726f0f29fa92 100644
--- a/.buildkite/pipelines/periodic.yml
+++ b/.buildkite/pipelines/periodic.yml
@@ -591,8 +591,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.13.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.5#bwcTest
+      - label: 8.13.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -601,7 +601,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.13.5
+          BWC_VERSION: 8.13.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -610,8 +610,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.14.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.0#bwcTest
+      - label: 8.14.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -620,7 +620,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.14.0
+          BWC_VERSION: 8.14.1
         retry:
           automatic:
             - exit_status: "-1"
@@ -714,7 +714,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.22", "8.13.5", "8.14.0", "8.15.0"]
+            BWC_VERSION: ["7.17.22", "8.14.1", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -762,7 +762,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.22", "8.13.5", "8.14.0", "8.15.0"]
+            BWC_VERSION: ["7.17.22", "8.14.1", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index b9afdcf23b858..3aa17cc370296 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -30,6 +30,6 @@ BWC_VERSION:
   - "8.10.4"
   - "8.11.4"
   - "8.12.2"
-  - "8.13.5"
-  - "8.14.0"
+  - "8.13.4"
+  - "8.14.1"
   - "8.15.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
index 213e4e93bc81d..f802829f6ec8a 100644
--- a/.ci/snapshotBwcVersions
+++ b/.ci/snapshotBwcVersions
@@ -1,5 +1,4 @@
 BWC_VERSION:
   - "7.17.22"
-  - "8.13.5"
-  - "8.14.0"
+  - "8.14.1"
   - "8.15.0"
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index dc161766b7954..06e4a1dd5368d 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -174,8 +174,8 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_13_2 = new Version(8_13_02_99);
     public static final Version V_8_13_3 = new Version(8_13_03_99);
     public static final Version V_8_13_4 = new Version(8_13_04_99);
-    public static final Version V_8_13_5 = new Version(8_13_05_99);
     public static final Version V_8_14_0 = new Version(8_14_00_99);
+    public static final Version V_8_14_1 = new Version(8_14_01_99);
     public static final Version V_8_15_0 = new Version(8_15_00_99);
     public static final Version CURRENT = V_8_15_0;
 
diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv
index 526f327b91c19..ef0c641bed04f 100644
--- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv
+++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv
@@ -120,3 +120,4 @@
 8.13.2,8595000
 8.13.3,8595000
 8.13.4,8595001
+8.14.0,8636001
diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
index 39f2a701726af..73f60f2e5ea7e 100644
--- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
+++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv
@@ -120,3 +120,4 @@
 8.13.2,8503000
 8.13.3,8503000
 8.13.4,8503000
+8.14.0,8505000

From b286921a36cc0f729771987e59ec0112d5597668 Mon Sep 17 00:00:00 2001
From: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co>
Date: Wed, 5 Jun 2024 15:07:52 +0000
Subject: [PATCH 169/208] Prune changelogs after 8.14.0 release

---
 docs/changelog/103542.yaml |  7 -------
 docs/changelog/104711.yaml |  5 -----
 docs/changelog/104830.yaml |  5 -----
 docs/changelog/104907.yaml |  6 ------
 docs/changelog/105063.yaml |  5 -----
 docs/changelog/105067.yaml |  5 -----
 docs/changelog/105168.yaml |  5 -----
 docs/changelog/105360.yaml |  6 ------
 docs/changelog/105393.yaml |  5 -----
 docs/changelog/105421.yaml |  5 -----
 docs/changelog/105439.yaml |  6 ------
 docs/changelog/105449.yaml |  6 ------
 docs/changelog/105454.yaml |  5 -----
 docs/changelog/105470.yaml |  5 -----
 docs/changelog/105477.yaml |  6 ------
 docs/changelog/105501.yaml |  5 -----
 docs/changelog/105517.yaml |  5 -----
 docs/changelog/105617.yaml |  5 -----
 docs/changelog/105622.yaml |  5 -----
 docs/changelog/105629.yaml |  5 -----
 docs/changelog/105636.yaml |  5 -----
 docs/changelog/105660.yaml |  5 -----
 docs/changelog/105670.yaml |  5 -----
 docs/changelog/105674.yaml |  6 ------
 docs/changelog/105689.yaml |  6 ------
 docs/changelog/105693.yaml |  6 ------
 docs/changelog/105709.yaml |  5 -----
 docs/changelog/105714.yaml |  5 -----
 docs/changelog/105717.yaml |  5 -----
 docs/changelog/105745.yaml |  6 ------
 docs/changelog/105757.yaml |  5 -----
 docs/changelog/105768.yaml |  5 -----
 docs/changelog/105779.yaml |  5 -----
 docs/changelog/105781.yaml |  5 -----
 docs/changelog/105791.yaml |  5 -----
 docs/changelog/105797.yaml |  5 -----
 docs/changelog/105847.yaml |  5 -----
 docs/changelog/105860.yaml |  5 -----
 docs/changelog/105893.yaml |  5 -----
 docs/changelog/105894.yaml |  5 -----
 docs/changelog/105985.yaml |  5 -----
 docs/changelog/106031.yaml | 13 -------------
 docs/changelog/106036.yaml |  5 -----
 docs/changelog/106053.yaml |  5 -----
 docs/changelog/106063.yaml |  5 -----
 docs/changelog/106065.yaml |  6 ------
 docs/changelog/106068.yaml | 21 ---------------------
 docs/changelog/106094.yaml |  5 -----
 docs/changelog/106102.yaml |  5 -----
 docs/changelog/106133.yaml | 19 -------------------
 docs/changelog/106150.yaml |  5 -----
 docs/changelog/106171.yaml |  6 ------
 docs/changelog/106172.yaml |  5 -----
 docs/changelog/106186.yaml |  6 ------
 docs/changelog/106189.yaml |  6 ------
 docs/changelog/106243.yaml |  5 -----
 docs/changelog/106244.yaml |  5 -----
 docs/changelog/106259.yaml |  5 -----
 docs/changelog/106285.yaml |  5 -----
 docs/changelog/106306.yaml |  6 ------
 docs/changelog/106315.yaml |  5 -----
 docs/changelog/106327.yaml |  5 -----
 docs/changelog/106338.yaml |  6 ------
 docs/changelog/106361.yaml |  5 -----
 docs/changelog/106373.yaml |  5 -----
 docs/changelog/106377.yaml |  5 -----
 docs/changelog/106378.yaml |  5 -----
 docs/changelog/106381.yaml |  5 -----
 docs/changelog/106396.yaml |  6 ------
 docs/changelog/106413.yaml |  6 ------
 docs/changelog/106429.yaml |  5 -----
 docs/changelog/106435.yaml |  6 ------
 docs/changelog/106472.yaml |  6 ------
 docs/changelog/106503.yaml |  5 -----
 docs/changelog/106511.yaml |  5 -----
 docs/changelog/106514.yaml |  6 ------
 docs/changelog/106516.yaml |  5 -----
 docs/changelog/106526.yaml |  5 -----
 docs/changelog/106531.yaml |  5 -----
 docs/changelog/106563.yaml |  5 -----
 docs/changelog/106575.yaml |  5 -----
 docs/changelog/106579.yaml |  5 -----
 docs/changelog/106638.yaml |  5 -----
 docs/changelog/106654.yaml |  6 ------
 docs/changelog/106685.yaml |  5 -----
 docs/changelog/106691.yaml |  6 ------
 docs/changelog/106708.yaml |  6 ------
 docs/changelog/106714.yaml |  5 -----
 docs/changelog/106720.yaml |  5 -----
 docs/changelog/106731.yaml |  5 -----
 docs/changelog/106745.yaml |  5 -----
 docs/changelog/106767.yaml |  5 -----
 docs/changelog/106796.yaml |  5 -----
 docs/changelog/106808.yaml |  5 -----
 docs/changelog/106810.yaml |  5 -----
 docs/changelog/106836.yaml |  5 -----
 docs/changelog/106840.yaml |  6 ------
 docs/changelog/106851.yaml |  5 -----
 docs/changelog/106852.yaml |  6 ------
 docs/changelog/106860.yaml |  5 -----
 docs/changelog/106862.yaml |  5 -----
 docs/changelog/106866.yaml |  5 -----
 docs/changelog/106889.yaml |  5 -----
 docs/changelog/106899.yaml |  6 ------
 docs/changelog/106919.yaml |  6 ------
 docs/changelog/106934.yaml |  5 -----
 docs/changelog/106952.yaml |  5 -----
 docs/changelog/106989.yaml |  7 -------
 docs/changelog/107007.yaml |  5 -----
 docs/changelog/107016.yaml |  5 -----
 docs/changelog/107038.yaml |  5 -----
 docs/changelog/107041.yaml |  6 ------
 docs/changelog/107046.yaml |  6 ------
 docs/changelog/107050.yaml |  5 -----
 docs/changelog/107107.yaml |  5 -----
 docs/changelog/107121.yaml |  6 ------
 docs/changelog/107129.yaml |  5 -----
 docs/changelog/107158.yaml |  5 -----
 docs/changelog/107178.yaml |  5 -----
 docs/changelog/107183.yaml |  5 -----
 docs/changelog/107196.yaml |  5 -----
 docs/changelog/107224.yaml |  6 ------
 docs/changelog/107232.yaml |  6 ------
 docs/changelog/107242.yaml |  5 -----
 docs/changelog/107253.yaml |  5 -----
 docs/changelog/107272.yaml |  5 -----
 docs/changelog/107287.yaml |  6 ------
 docs/changelog/107291.yaml |  6 ------
 docs/changelog/107303.yaml |  5 -----
 docs/changelog/107312.yaml |  5 -----
 docs/changelog/107334.yaml |  5 -----
 docs/changelog/107358.yaml |  6 ------
 docs/changelog/107370.yaml |  5 -----
 docs/changelog/107377.yaml | 13 -------------
 docs/changelog/107383.yaml |  6 ------
 docs/changelog/107411.yaml |  5 -----
 docs/changelog/107414.yaml |  7 -------
 docs/changelog/107447.yaml |  5 -----
 docs/changelog/107449.yaml |  5 -----
 docs/changelog/107467.yaml |  5 -----
 docs/changelog/107494.yaml |  6 ------
 docs/changelog/107517.yaml |  5 -----
 docs/changelog/107533.yaml |  5 -----
 docs/changelog/107551.yaml |  5 -----
 docs/changelog/107577.yaml |  6 ------
 docs/changelog/107578.yaml |  5 -----
 docs/changelog/107598.yaml |  5 -----
 docs/changelog/107655.yaml |  6 ------
 docs/changelog/107678.yaml |  6 ------
 docs/changelog/107743.yaml |  5 -----
 docs/changelog/107828.yaml |  6 ------
 docs/changelog/107865.yaml |  5 -----
 docs/changelog/107891.yaml |  6 ------
 docs/changelog/107902.yaml |  5 -----
 docs/changelog/107969.yaml |  5 -----
 docs/changelog/108007.yaml |  5 -----
 docs/changelog/108031.yaml |  6 ------
 docs/changelog/108041.yaml |  7 -------
 docs/changelog/108101.yaml |  6 ------
 docs/changelog/108238.yaml |  6 ------
 docs/changelog/108257.yaml |  5 -----
 docs/changelog/108365.yaml |  5 -----
 docs/changelog/108431.yaml |  5 -----
 docs/changelog/108518.yaml |  5 -----
 docs/changelog/108562.yaml |  6 ------
 docs/changelog/108571.yaml |  5 -----
 docs/changelog/108600.yaml | 15 ---------------
 docs/changelog/108654.yaml |  5 -----
 docs/changelog/108736.yaml |  5 -----
 docs/changelog/108802.yaml |  5 -----
 docs/changelog/108834.yaml |  6 ------
 docs/changelog/108854.yaml |  5 -----
 docs/changelog/108867.yaml |  6 ------
 docs/changelog/108900.yaml |  6 ------
 docs/changelog/109020.yaml |  6 ------
 docs/changelog/109034.yaml |  5 -----
 docs/changelog/109048.yaml |  6 ------
 docs/changelog/109097.yaml |  6 ------
 docs/changelog/109148.yaml |  6 ------
 docs/changelog/109173.yaml |  5 -----
 docs/changelog/97072.yaml  |  5 -----
 docs/changelog/97561.yaml  |  5 -----
 docs/changelog/99048.yaml  |  6 ------
 183 files changed, 1033 deletions(-)
 delete mode 100644 docs/changelog/103542.yaml
 delete mode 100644 docs/changelog/104711.yaml
 delete mode 100644 docs/changelog/104830.yaml
 delete mode 100644 docs/changelog/104907.yaml
 delete mode 100644 docs/changelog/105063.yaml
 delete mode 100644 docs/changelog/105067.yaml
 delete mode 100644 docs/changelog/105168.yaml
 delete mode 100644 docs/changelog/105360.yaml
 delete mode 100644 docs/changelog/105393.yaml
 delete mode 100644 docs/changelog/105421.yaml
 delete mode 100644 docs/changelog/105439.yaml
 delete mode 100644 docs/changelog/105449.yaml
 delete mode 100644 docs/changelog/105454.yaml
 delete mode 100644 docs/changelog/105470.yaml
 delete mode 100644 docs/changelog/105477.yaml
 delete mode 100644 docs/changelog/105501.yaml
 delete mode 100644 docs/changelog/105517.yaml
 delete mode 100644 docs/changelog/105617.yaml
 delete mode 100644 docs/changelog/105622.yaml
 delete mode 100644 docs/changelog/105629.yaml
 delete mode 100644 docs/changelog/105636.yaml
 delete mode 100644 docs/changelog/105660.yaml
 delete mode 100644 docs/changelog/105670.yaml
 delete mode 100644 docs/changelog/105674.yaml
 delete mode 100644 docs/changelog/105689.yaml
 delete mode 100644 docs/changelog/105693.yaml
 delete mode 100644 docs/changelog/105709.yaml
 delete mode 100644 docs/changelog/105714.yaml
 delete mode 100644 docs/changelog/105717.yaml
 delete mode 100644 docs/changelog/105745.yaml
 delete mode 100644 docs/changelog/105757.yaml
 delete mode 100644 docs/changelog/105768.yaml
 delete mode 100644 docs/changelog/105779.yaml
 delete mode 100644 docs/changelog/105781.yaml
 delete mode 100644 docs/changelog/105791.yaml
 delete mode 100644 docs/changelog/105797.yaml
 delete mode 100644 docs/changelog/105847.yaml
 delete mode 100644 docs/changelog/105860.yaml
 delete mode 100644 docs/changelog/105893.yaml
 delete mode 100644 docs/changelog/105894.yaml
 delete mode 100644 docs/changelog/105985.yaml
 delete mode 100644 docs/changelog/106031.yaml
 delete mode 100644 docs/changelog/106036.yaml
 delete mode 100644 docs/changelog/106053.yaml
 delete mode 100644 docs/changelog/106063.yaml
 delete mode 100644 docs/changelog/106065.yaml
 delete mode 100644 docs/changelog/106068.yaml
 delete mode 100644 docs/changelog/106094.yaml
 delete mode 100644 docs/changelog/106102.yaml
 delete mode 100644 docs/changelog/106133.yaml
 delete mode 100644 docs/changelog/106150.yaml
 delete mode 100644 docs/changelog/106171.yaml
 delete mode 100644 docs/changelog/106172.yaml
 delete mode 100644 docs/changelog/106186.yaml
 delete mode 100644 docs/changelog/106189.yaml
 delete mode 100644 docs/changelog/106243.yaml
 delete mode 100644 docs/changelog/106244.yaml
 delete mode 100644 docs/changelog/106259.yaml
 delete mode 100644 docs/changelog/106285.yaml
 delete mode 100644 docs/changelog/106306.yaml
 delete mode 100644 docs/changelog/106315.yaml
 delete mode 100644 docs/changelog/106327.yaml
 delete mode 100644 docs/changelog/106338.yaml
 delete mode 100644 docs/changelog/106361.yaml
 delete mode 100644 docs/changelog/106373.yaml
 delete mode 100644 docs/changelog/106377.yaml
 delete mode 100644 docs/changelog/106378.yaml
 delete mode 100644 docs/changelog/106381.yaml
 delete mode 100644 docs/changelog/106396.yaml
 delete mode 100644 docs/changelog/106413.yaml
 delete mode 100644 docs/changelog/106429.yaml
 delete mode 100644 docs/changelog/106435.yaml
 delete mode 100644 docs/changelog/106472.yaml
 delete mode 100644 docs/changelog/106503.yaml
 delete mode 100644 docs/changelog/106511.yaml
 delete mode 100644 docs/changelog/106514.yaml
 delete mode 100644 docs/changelog/106516.yaml
 delete mode 100644 docs/changelog/106526.yaml
 delete mode 100644 docs/changelog/106531.yaml
 delete mode 100644 docs/changelog/106563.yaml
 delete mode 100644 docs/changelog/106575.yaml
 delete mode 100644 docs/changelog/106579.yaml
 delete mode 100644 docs/changelog/106638.yaml
 delete mode 100644 docs/changelog/106654.yaml
 delete mode 100644 docs/changelog/106685.yaml
 delete mode 100644 docs/changelog/106691.yaml
 delete mode 100644 docs/changelog/106708.yaml
 delete mode 100644 docs/changelog/106714.yaml
 delete mode 100644 docs/changelog/106720.yaml
 delete mode 100644 docs/changelog/106731.yaml
 delete mode 100644 docs/changelog/106745.yaml
 delete mode 100644 docs/changelog/106767.yaml
 delete mode 100644 docs/changelog/106796.yaml
 delete mode 100644 docs/changelog/106808.yaml
 delete mode 100644 docs/changelog/106810.yaml
 delete mode 100644 docs/changelog/106836.yaml
 delete mode 100644 docs/changelog/106840.yaml
 delete mode 100644 docs/changelog/106851.yaml
 delete mode 100644 docs/changelog/106852.yaml
 delete mode 100644 docs/changelog/106860.yaml
 delete mode 100644 docs/changelog/106862.yaml
 delete mode 100644 docs/changelog/106866.yaml
 delete mode 100644 docs/changelog/106889.yaml
 delete mode 100644 docs/changelog/106899.yaml
 delete mode 100644 docs/changelog/106919.yaml
 delete mode 100644 docs/changelog/106934.yaml
 delete mode 100644 docs/changelog/106952.yaml
 delete mode 100644 docs/changelog/106989.yaml
 delete mode 100644 docs/changelog/107007.yaml
 delete mode 100644 docs/changelog/107016.yaml
 delete mode 100644 docs/changelog/107038.yaml
 delete mode 100644 docs/changelog/107041.yaml
 delete mode 100644 docs/changelog/107046.yaml
 delete mode 100644 docs/changelog/107050.yaml
 delete mode 100644 docs/changelog/107107.yaml
 delete mode 100644 docs/changelog/107121.yaml
 delete mode 100644 docs/changelog/107129.yaml
 delete mode 100644 docs/changelog/107158.yaml
 delete mode 100644 docs/changelog/107178.yaml
 delete mode 100644 docs/changelog/107183.yaml
 delete mode 100644 docs/changelog/107196.yaml
 delete mode 100644 docs/changelog/107224.yaml
 delete mode 100644 docs/changelog/107232.yaml
 delete mode 100644 docs/changelog/107242.yaml
 delete mode 100644 docs/changelog/107253.yaml
 delete mode 100644 docs/changelog/107272.yaml
 delete mode 100644 docs/changelog/107287.yaml
 delete mode 100644 docs/changelog/107291.yaml
 delete mode 100644 docs/changelog/107303.yaml
 delete mode 100644 docs/changelog/107312.yaml
 delete mode 100644 docs/changelog/107334.yaml
 delete mode 100644 docs/changelog/107358.yaml
 delete mode 100644 docs/changelog/107370.yaml
 delete mode 100644 docs/changelog/107377.yaml
 delete mode 100644 docs/changelog/107383.yaml
 delete mode 100644 docs/changelog/107411.yaml
 delete mode 100644 docs/changelog/107414.yaml
 delete mode 100644 docs/changelog/107447.yaml
 delete mode 100644 docs/changelog/107449.yaml
 delete mode 100644 docs/changelog/107467.yaml
 delete mode 100644 docs/changelog/107494.yaml
 delete mode 100644 docs/changelog/107517.yaml
 delete mode 100644 docs/changelog/107533.yaml
 delete mode 100644 docs/changelog/107551.yaml
 delete mode 100644 docs/changelog/107577.yaml
 delete mode 100644 docs/changelog/107578.yaml
 delete mode 100644 docs/changelog/107598.yaml
 delete mode 100644 docs/changelog/107655.yaml
 delete mode 100644 docs/changelog/107678.yaml
 delete mode 100644 docs/changelog/107743.yaml
 delete mode 100644 docs/changelog/107828.yaml
 delete mode 100644 docs/changelog/107865.yaml
 delete mode 100644 docs/changelog/107891.yaml
 delete mode 100644 docs/changelog/107902.yaml
 delete mode 100644 docs/changelog/107969.yaml
 delete mode 100644 docs/changelog/108007.yaml
 delete mode 100644 docs/changelog/108031.yaml
 delete mode 100644 docs/changelog/108041.yaml
 delete mode 100644 docs/changelog/108101.yaml
 delete mode 100644 docs/changelog/108238.yaml
 delete mode 100644 docs/changelog/108257.yaml
 delete mode 100644 docs/changelog/108365.yaml
 delete mode 100644 docs/changelog/108431.yaml
 delete mode 100644 docs/changelog/108518.yaml
 delete mode 100644 docs/changelog/108562.yaml
 delete mode 100644 docs/changelog/108571.yaml
 delete mode 100644 docs/changelog/108600.yaml
 delete mode 100644 docs/changelog/108654.yaml
 delete mode 100644 docs/changelog/108736.yaml
 delete mode 100644 docs/changelog/108802.yaml
 delete mode 100644 docs/changelog/108834.yaml
 delete mode 100644 docs/changelog/108854.yaml
 delete mode 100644 docs/changelog/108867.yaml
 delete mode 100644 docs/changelog/108900.yaml
 delete mode 100644 docs/changelog/109020.yaml
 delete mode 100644 docs/changelog/109034.yaml
 delete mode 100644 docs/changelog/109048.yaml
 delete mode 100644 docs/changelog/109097.yaml
 delete mode 100644 docs/changelog/109148.yaml
 delete mode 100644 docs/changelog/109173.yaml
 delete mode 100644 docs/changelog/97072.yaml
 delete mode 100644 docs/changelog/97561.yaml
 delete mode 100644 docs/changelog/99048.yaml

diff --git a/docs/changelog/103542.yaml b/docs/changelog/103542.yaml
deleted file mode 100644
index 74e713eb2f606..0000000000000
--- a/docs/changelog/103542.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-pr: 103542
-summary: Flatten object mappings when subobjects is false
-area: Mapping
-type: feature
-issues:
- - 99860
- - 103497
diff --git a/docs/changelog/104711.yaml b/docs/changelog/104711.yaml
deleted file mode 100644
index f0f9bf7f10e45..0000000000000
--- a/docs/changelog/104711.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 104711
-summary: "Fixing NPE when requesting [_none_] for `stored_fields`"
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/104830.yaml b/docs/changelog/104830.yaml
deleted file mode 100644
index c056f3d618b75..0000000000000
--- a/docs/changelog/104830.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 104830
-summary: All new `shard_seed` parameter for `random_sampler` agg
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/104907.yaml b/docs/changelog/104907.yaml
deleted file mode 100644
index 0d8592ae29526..0000000000000
--- a/docs/changelog/104907.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 104907
-summary: Support ST_INTERSECTS between geometry column and other geometry or string
-area: "ES|QL"
-type: enhancement
-issues:
-- 104874
diff --git a/docs/changelog/105063.yaml b/docs/changelog/105063.yaml
deleted file mode 100644
index 668f8ac104493..0000000000000
--- a/docs/changelog/105063.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105063
-summary: Infrastructure for metering the update requests
-area: Infra/Metrics
-type: enhancement
-issues: []
diff --git a/docs/changelog/105067.yaml b/docs/changelog/105067.yaml
deleted file mode 100644
index 562e8271f5502..0000000000000
--- a/docs/changelog/105067.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105067
-summary: "ESQL: Use faster field caps"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/105168.yaml b/docs/changelog/105168.yaml
deleted file mode 100644
index 0f3792b832f55..0000000000000
--- a/docs/changelog/105168.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105168
-summary: Add ?master_timeout query parameter to ccr apis
-area: CCR
-type: bug
-issues: []
diff --git a/docs/changelog/105360.yaml b/docs/changelog/105360.yaml
deleted file mode 100644
index 41a7ea24e5500..0000000000000
--- a/docs/changelog/105360.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 105360
-summary: Cross-cluster painless/execute actions should check permissions only on target
-  remote cluster
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/105393.yaml b/docs/changelog/105393.yaml
deleted file mode 100644
index 4a4cc299b7bd7..0000000000000
--- a/docs/changelog/105393.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105393
-summary: Adding support for hex-encoded byte vectors on knn-search
-area: Vector Search
-type: feature
-issues: []
diff --git a/docs/changelog/105421.yaml b/docs/changelog/105421.yaml
deleted file mode 100644
index 2ff9ef008c803..0000000000000
--- a/docs/changelog/105421.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105421
-summary: "ESQL: Add timers to many status results"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/105439.yaml b/docs/changelog/105439.yaml
deleted file mode 100644
index 45bbede469542..0000000000000
--- a/docs/changelog/105439.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 105439
-summary: Support Profile Activate with JWTs with client authn
-area: Authentication
-type: enhancement
-issues:
- - 105342
diff --git a/docs/changelog/105449.yaml b/docs/changelog/105449.yaml
deleted file mode 100644
index b565d6c782bd9..0000000000000
--- a/docs/changelog/105449.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 105449
-summary: Don't stop checking if the `HealthNode` persistent task is present
-area: Health
-type: bug
-issues:
- - 98926
diff --git a/docs/changelog/105454.yaml b/docs/changelog/105454.yaml
deleted file mode 100644
index fc814a343c46b..0000000000000
--- a/docs/changelog/105454.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105454
-summary: "ESQL: Sum of constants"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/105470.yaml b/docs/changelog/105470.yaml
deleted file mode 100644
index 56425de6c88e4..0000000000000
--- a/docs/changelog/105470.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105470
-summary: Add retrievers using the parser-only approach
-area: Ranking
-type: enhancement
-issues: []
diff --git a/docs/changelog/105477.yaml b/docs/changelog/105477.yaml
deleted file mode 100644
index f994d38a3f671..0000000000000
--- a/docs/changelog/105477.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 105477
-summary: "ESQL: Introduce expression validation phase"
-area: ES|QL
-type: enhancement
-issues:
- - 105425
diff --git a/docs/changelog/105501.yaml b/docs/changelog/105501.yaml
deleted file mode 100644
index 2e5e375764640..0000000000000
--- a/docs/changelog/105501.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105501
-summary: Support non-keyword dimensions as routing fields in TSDB
-area: TSDB
-type: enhancement
-issues: []
diff --git a/docs/changelog/105517.yaml b/docs/changelog/105517.yaml
deleted file mode 100644
index 7cca86d1cff6e..0000000000000
--- a/docs/changelog/105517.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105517
-summary: Upgrade to Netty 4.1.107
-area: Network
-type: upgrade
-issues: []
diff --git a/docs/changelog/105617.yaml b/docs/changelog/105617.yaml
deleted file mode 100644
index 7fd8203336fff..0000000000000
--- a/docs/changelog/105617.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105617
-summary: Fix HTTP corner-case response leaks
-area: Network
-type: bug
-issues: []
diff --git a/docs/changelog/105622.yaml b/docs/changelog/105622.yaml
deleted file mode 100644
index 33093f5ffceb5..0000000000000
--- a/docs/changelog/105622.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105622
-summary: Distinguish different snapshot failures by log level
-area: Snapshot/Restore
-type: enhancement
-issues: []
diff --git a/docs/changelog/105629.yaml b/docs/changelog/105629.yaml
deleted file mode 100644
index 00fa73a759558..0000000000000
--- a/docs/changelog/105629.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105629
-summary: Show owner `realm_type` for returned API keys
-area: Security
-type: enhancement
-issues: []
diff --git a/docs/changelog/105636.yaml b/docs/changelog/105636.yaml
deleted file mode 100644
index 01f27199771d4..0000000000000
--- a/docs/changelog/105636.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105636
-summary: Flip dynamic mapping condition when create tsid
-area: TSDB
-type: bug
-issues: []
diff --git a/docs/changelog/105660.yaml b/docs/changelog/105660.yaml
deleted file mode 100644
index 1b30a25417906..0000000000000
--- a/docs/changelog/105660.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105660
-summary: "Text structure endpoints to determine the structure of a list of messages and of an indexed field"
-area: Machine Learning
-type: feature
-issues: []
diff --git a/docs/changelog/105670.yaml b/docs/changelog/105670.yaml
deleted file mode 100644
index 234f4b6af5a73..0000000000000
--- a/docs/changelog/105670.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105670
-summary: "Painless: Apply true regex limit factor with FIND and MATCH operation"
-area: Infra/Scripting
-type: bug
-issues: []
diff --git a/docs/changelog/105674.yaml b/docs/changelog/105674.yaml
deleted file mode 100644
index 7b8d04f4687a3..0000000000000
--- a/docs/changelog/105674.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 105674
-summary: Health monitor concurrency fixes
-area: Health
-type: bug
-issues:
- - 105065
diff --git a/docs/changelog/105689.yaml b/docs/changelog/105689.yaml
deleted file mode 100644
index e76281f1b2fc7..0000000000000
--- a/docs/changelog/105689.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 105689
-summary: Fix `uri_parts` processor behaviour for missing extensions
-area: Ingest Node
-type: bug
-issues:
- - 105612
diff --git a/docs/changelog/105693.yaml b/docs/changelog/105693.yaml
deleted file mode 100644
index 8d14d611e19a3..0000000000000
--- a/docs/changelog/105693.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 105693
-summary: Fix error 500 on invalid `ParentIdQuery`
-area: Search
-type: bug
-issues:
- - 105366
diff --git a/docs/changelog/105709.yaml b/docs/changelog/105709.yaml
deleted file mode 100644
index 41b6e749d9270..0000000000000
--- a/docs/changelog/105709.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105709
-summary: Apply stricter Document Level Security (DLS) rules for the validate query API with the rewrite parameter.
-area: Security
-type: bug
-issues: []
diff --git a/docs/changelog/105714.yaml b/docs/changelog/105714.yaml
deleted file mode 100644
index b6ab5e128c72c..0000000000000
--- a/docs/changelog/105714.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105714
-summary: Apply stricter Document Level Security (DLS) rules for terms aggregations when min_doc_count is set to 0.
-area: "Aggregations"
-type: bug
-issues: []
diff --git a/docs/changelog/105717.yaml b/docs/changelog/105717.yaml
deleted file mode 100644
index c75bc4fe65798..0000000000000
--- a/docs/changelog/105717.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105717
-summary: Upgrade jna to 5.12.1
-area: Infra/Core
-type: upgrade
-issues: []
diff --git a/docs/changelog/105745.yaml b/docs/changelog/105745.yaml
deleted file mode 100644
index e9a61f692d94d..0000000000000
--- a/docs/changelog/105745.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 105745
-summary: Fix `noop_update_total` is not being updated when using the `_bulk`
-area: CRUD
-type: bug
-issues:
-  - 105742
diff --git a/docs/changelog/105757.yaml b/docs/changelog/105757.yaml
deleted file mode 100644
index f11aed2b2d96b..0000000000000
--- a/docs/changelog/105757.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105757
-summary: Add pluggable `BuildVersion` in `NodeMetadata`
-area: Infra/Core
-type: enhancement
-issues: []
diff --git a/docs/changelog/105768.yaml b/docs/changelog/105768.yaml
deleted file mode 100644
index 49d7f1f15c453..0000000000000
--- a/docs/changelog/105768.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105768
-summary: Add two new OGC functions ST_X and ST_Y
-area: "ES|QL"
-type: enhancement
-issues: []
diff --git a/docs/changelog/105779.yaml b/docs/changelog/105779.yaml
deleted file mode 100644
index 3699ca0e2f246..0000000000000
--- a/docs/changelog/105779.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105779
-summary: "[Profiling] Speed up serialization of flamegraph"
-area: Application
-type: enhancement
-issues: []
diff --git a/docs/changelog/105781.yaml b/docs/changelog/105781.yaml
deleted file mode 100644
index c3ae7f0035904..0000000000000
--- a/docs/changelog/105781.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105781
-summary: CCS with `minimize_roundtrips` performs incremental merges of each `SearchResponse`
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/105791.yaml b/docs/changelog/105791.yaml
deleted file mode 100644
index f18b5e6b8fdd7..0000000000000
--- a/docs/changelog/105791.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105791
-summary: "Bugfix: Disable eager loading `BitSetFilterCache` on Indexing Nodes"
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/105797.yaml b/docs/changelog/105797.yaml
deleted file mode 100644
index 7c832e2e5e63c..0000000000000
--- a/docs/changelog/105797.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105797
-summary: Enable retrying on 500 error response from Cohere text embedding API
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/105847.yaml b/docs/changelog/105847.yaml
deleted file mode 100644
index a731395bc9a81..0000000000000
--- a/docs/changelog/105847.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105847
-summary: (API+) CAT Nodes alias for shard header to match CAT Allocation
-area: Stats
-type: enhancement
-issues: []
diff --git a/docs/changelog/105860.yaml b/docs/changelog/105860.yaml
deleted file mode 100644
index 71f3544a02a1f..0000000000000
--- a/docs/changelog/105860.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105860
-summary: "ESQL: Re-enable logical dependency check"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/105893.yaml b/docs/changelog/105893.yaml
deleted file mode 100644
index c88736f5dda3d..0000000000000
--- a/docs/changelog/105893.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105893
-summary: Specialize serialization for `ArrayVectors`
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/105894.yaml b/docs/changelog/105894.yaml
deleted file mode 100644
index a1a99eaa6259b..0000000000000
--- a/docs/changelog/105894.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105894
-summary: Add allocation stats
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/105985.yaml b/docs/changelog/105985.yaml
deleted file mode 100644
index 2f2a8c1394070..0000000000000
--- a/docs/changelog/105985.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 105985
-summary: Wait forever for `IndexTemplateRegistry` asset installation
-area: Indices APIs
-type: enhancement
-issues: []
diff --git a/docs/changelog/106031.yaml b/docs/changelog/106031.yaml
deleted file mode 100644
index d0a0303e74164..0000000000000
--- a/docs/changelog/106031.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-pr: 106031
-summary: Deprecate allowing `fields` in scenarios where it is ignored
-area: Mapping
-type: deprecation
-issues: []
-deprecation:
-  title: Deprecate allowing `fields` in scenarios where it is ignored
-  area: Mapping
-  details: The following mapped types have always ignored `fields` when using multi-fields.
-    This deprecation makes this clearer and we will completely disallow `fields` for
-    these mapped types in the future.
-  impact: "In the future, `join`, `aggregate_metric_double`, and `constant_keyword`,\
-    \ will all disallow supplying `fields` as a parameter in the mapping."
diff --git a/docs/changelog/106036.yaml b/docs/changelog/106036.yaml
deleted file mode 100644
index 7b129c6c0a7a3..0000000000000
--- a/docs/changelog/106036.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106036
-summary: Add status for enrich operator
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106053.yaml b/docs/changelog/106053.yaml
deleted file mode 100644
index 72cfe0207795d..0000000000000
--- a/docs/changelog/106053.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106053
-summary: Speed up serialization of `BytesRefArray`
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106063.yaml b/docs/changelog/106063.yaml
deleted file mode 100644
index 57c05370a943f..0000000000000
--- a/docs/changelog/106063.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106063
-summary: Consider `ShardRouting` roles when calculating shard copies in shutdown status
-area: Infra/Node Lifecycle
-type: bug
-issues: []
diff --git a/docs/changelog/106065.yaml b/docs/changelog/106065.yaml
deleted file mode 100644
index b87f4848fb574..0000000000000
--- a/docs/changelog/106065.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106065
-summary: "ESQL: Values aggregation function"
-area: ES|QL
-type: feature
-issues:
- - 103600
diff --git a/docs/changelog/106068.yaml b/docs/changelog/106068.yaml
deleted file mode 100644
index 51bcc2bcf98b0..0000000000000
--- a/docs/changelog/106068.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-pr: 106068
-summary: Add `modelId` and `modelText` to `KnnVectorQueryBuilder`
-area: Search
-type: enhancement
-issues: []
-highlight:
-  title: Query phase KNN now supports query_vector_builder
-  body: |-
-    It is now possible to pass `model_text` and `model_id` within a `knn` query
-    in the [query DSL](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-knn-query.html) to convert a text query into a dense vector and run the
-    nearest neighbor query on it, instead of requiring the dense vector to be
-    directly passed (within the `query_vector` parameter). Similar to the
-    [top-level knn query](https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html) (executed in the DFS phase), it is possible to supply
-    a `query_vector_builder` object containing a `text_embedding` object with
-    `model_text` (the text query to be converted into a dense vector) and
-    `model_id` (the identifier of a deployed model responsible for transforming
-    the text query into a dense vector). Note that an embedding model with the
-    referenced `model_id` needs to be [deployed on a ML node](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-deploy-models.html).
-    in the cluster.
-  notable: true
-
diff --git a/docs/changelog/106094.yaml b/docs/changelog/106094.yaml
deleted file mode 100644
index 4341164222338..0000000000000
--- a/docs/changelog/106094.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106094
-summary: "ESQL: Support partially folding CASE"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106102.yaml b/docs/changelog/106102.yaml
deleted file mode 100644
index b7c13514f6715..0000000000000
--- a/docs/changelog/106102.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106102
-summary: Specialize serialization of array blocks
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106133.yaml b/docs/changelog/106133.yaml
deleted file mode 100644
index 6dd7bf6cea086..0000000000000
--- a/docs/changelog/106133.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-pr: 106133
-summary: Add a SIMD (Neon) optimised vector distance function for int8
-area: Search
-type: enhancement
-issues: []
-highlight:
-  title: A SIMD (Neon) optimised vector distance function for merging int8 Scalar Quantized vectors has been added
-  body: |-
-    An optimised int8 vector distance implementation for aarch64 has been added.
-    This implementation is currently only used during merging.
-    The vector distance implementation outperforms Lucene's Pamana Vector
-    implementation for binary comparisons by approx 5x (depending on the number
-    of dimensions). It does so by means of SIMD (Neon) intrinsics compiled into a
-    separate native library and link by Panama's FFI. Comparisons are performed on
-    off-heap mmap'ed vector data.
-    Macro benchmarks, SO_Dense_Vector with scalar quantization enabled, shows
-    significant improvements in merge times, approximately 3 times faster. 
-  notable: true
-  
diff --git a/docs/changelog/106150.yaml b/docs/changelog/106150.yaml
deleted file mode 100644
index 05bd8b06987c6..0000000000000
--- a/docs/changelog/106150.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106150
-summary: Use correct system index bulk executor
-area: CRUD
-type: bug
-issues: []
diff --git a/docs/changelog/106171.yaml b/docs/changelog/106171.yaml
deleted file mode 100644
index 9daf1b9acd994..0000000000000
--- a/docs/changelog/106171.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106171
-summary: Do not log error on node restart when the transform is already failed
-area: Transform
-type: enhancement
-issues:
- - 106168
diff --git a/docs/changelog/106172.yaml b/docs/changelog/106172.yaml
deleted file mode 100644
index 80d80b9d7f299..0000000000000
--- a/docs/changelog/106172.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106172
-summary: "[Profiling] Allow to override index settings"
-area: Application
-type: enhancement
-issues: []
diff --git a/docs/changelog/106186.yaml b/docs/changelog/106186.yaml
deleted file mode 100644
index 097639dd28f1b..0000000000000
--- a/docs/changelog/106186.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106186
-summary: Expand support for ENRICH to full set supported by ES ingest processors
-area: ES|QL
-type: enhancement
-issues:
- - 106162
diff --git a/docs/changelog/106189.yaml b/docs/changelog/106189.yaml
deleted file mode 100644
index ec485f0e60efb..0000000000000
--- a/docs/changelog/106189.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106189
-summary: Fix numeric sorts in `_cat/nodes`
-area: CAT APIs
-type: bug
-issues:
- - 48070
diff --git a/docs/changelog/106243.yaml b/docs/changelog/106243.yaml
deleted file mode 100644
index 6b02e3f1699d4..0000000000000
--- a/docs/changelog/106243.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106243
-summary: "[Transform] Auto retry Transform start"
-area: "Transform"
-type: bug
-issues: []
diff --git a/docs/changelog/106244.yaml b/docs/changelog/106244.yaml
deleted file mode 100644
index fe03f575b9efb..0000000000000
--- a/docs/changelog/106244.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106244
-summary: Support ES|QL requests through the `NodeClient::execute`
-area: ES|QL
-type: feature
-issues: []
diff --git a/docs/changelog/106259.yaml b/docs/changelog/106259.yaml
deleted file mode 100644
index d56b5e5a5e379..0000000000000
--- a/docs/changelog/106259.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106259
-summary: Add data stream lifecycle to kibana reporting template
-area: Data streams
-type: enhancement
-issues: []
diff --git a/docs/changelog/106285.yaml b/docs/changelog/106285.yaml
deleted file mode 100644
index 37a7e67fe9395..0000000000000
--- a/docs/changelog/106285.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106285
-summary: Add a check for the same feature being declared regular and historical
-area: Infra/Core
-type: bug
-issues: []
diff --git a/docs/changelog/106306.yaml b/docs/changelog/106306.yaml
deleted file mode 100644
index 571fe73c31a3e..0000000000000
--- a/docs/changelog/106306.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 99961
-summary: "added fix for inconsistent text trimming in Unified Highlighter"
-area: Highlighting
-type: bug
-issues:
- - 101803
diff --git a/docs/changelog/106315.yaml b/docs/changelog/106315.yaml
deleted file mode 100644
index 57c41c8024d20..0000000000000
--- a/docs/changelog/106315.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106315
-summary: Updating the tika version to 2.9.1 in the ingest attachment plugin
-area: Ingest Node
-type: upgrade
-issues: []
diff --git a/docs/changelog/106327.yaml b/docs/changelog/106327.yaml
deleted file mode 100644
index 2b4b811ece40b..0000000000000
--- a/docs/changelog/106327.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106327
-summary: Serialize big array vectors
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106338.yaml b/docs/changelog/106338.yaml
deleted file mode 100644
index c05826d87a11f..0000000000000
--- a/docs/changelog/106338.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106338
-summary: Text fields are stored by default in TSDB indices
-area: TSDB
-type: enhancement
-issues:
- - 97039
diff --git a/docs/changelog/106361.yaml b/docs/changelog/106361.yaml
deleted file mode 100644
index a4cd608279c12..0000000000000
--- a/docs/changelog/106361.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106361
-summary: Add a `PriorityQueue` backed by `BigArrays`
-area: Aggregations
-type: enhancement
-issues: []
diff --git a/docs/changelog/106373.yaml b/docs/changelog/106373.yaml
deleted file mode 100644
index e838c7b1a660d..0000000000000
--- a/docs/changelog/106373.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106373
-summary: Serialize big array blocks
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106377.yaml b/docs/changelog/106377.yaml
deleted file mode 100644
index 7f0f18d43b440..0000000000000
--- a/docs/changelog/106377.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106377
-summary: Add transport version for search load autoscaling
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/106378.yaml b/docs/changelog/106378.yaml
deleted file mode 100644
index b54760553d184..0000000000000
--- a/docs/changelog/106378.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106378
-summary: Add Cohere rerank to `_inference` service
-area: Machine Learning
-type: feature
-issues: []
diff --git a/docs/changelog/106381.yaml b/docs/changelog/106381.yaml
deleted file mode 100644
index 500f6d5416822..0000000000000
--- a/docs/changelog/106381.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106381
-summary: Dedupe terms in terms queries
-area: Mapping
-type: bug
-issues: []
diff --git a/docs/changelog/106396.yaml b/docs/changelog/106396.yaml
deleted file mode 100644
index 7aa06566c75e7..0000000000000
--- a/docs/changelog/106396.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106396
-summary: "Check preTags and postTags params for empty values"
-area: Highlighting
-type: bug
-issues:
-  - 69009
diff --git a/docs/changelog/106413.yaml b/docs/changelog/106413.yaml
deleted file mode 100644
index 8e13a839bc41e..0000000000000
--- a/docs/changelog/106413.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106413
-summary: Consolidate permissions checks
-area: Transform
-type: bug
-issues:
- - 105794
diff --git a/docs/changelog/106429.yaml b/docs/changelog/106429.yaml
deleted file mode 100644
index 7ac524d13909b..0000000000000
--- a/docs/changelog/106429.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106429
-summary: "ESQL: Regex improvements"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106435.yaml b/docs/changelog/106435.yaml
deleted file mode 100644
index 5bfe0087a93d3..0000000000000
--- a/docs/changelog/106435.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106435
-summary: "ENRICH support for TEXT fields"
-area: ES|QL
-type: enhancement
-issues:
- - 105384
diff --git a/docs/changelog/106472.yaml b/docs/changelog/106472.yaml
deleted file mode 100644
index 120286c4cd8c7..0000000000000
--- a/docs/changelog/106472.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106472
-summary: "Fix the position of spike, dip and distribution changes bucket when the\
-  \ sibling aggregation includes empty buckets"
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/106503.yaml b/docs/changelog/106503.yaml
deleted file mode 100644
index 1b7e78d8ffc27..0000000000000
--- a/docs/changelog/106503.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106503
-summary: "Support ST_CONTAINS and ST_WITHIN"
-area: "ES|QL"
-type: enhancement
-issues: []
diff --git a/docs/changelog/106511.yaml b/docs/changelog/106511.yaml
deleted file mode 100644
index bdef7f1aea225..0000000000000
--- a/docs/changelog/106511.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106511
-summary: Wait indefintely for http connections on shutdown by default
-area: Infra/Node Lifecycle
-type: bug
-issues: []
diff --git a/docs/changelog/106514.yaml b/docs/changelog/106514.yaml
deleted file mode 100644
index 5b25f40db2742..0000000000000
--- a/docs/changelog/106514.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106514
-summary: Add granular error list to alias action response
-area: Indices APIs
-type: feature
-issues:
- - 94478
diff --git a/docs/changelog/106516.yaml b/docs/changelog/106516.yaml
deleted file mode 100644
index 905896fb0ef03..0000000000000
--- a/docs/changelog/106516.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106516
-summary: "ESQL: perform a reduction on the data node"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106526.yaml b/docs/changelog/106526.yaml
deleted file mode 100644
index ac98454b5d8b4..0000000000000
--- a/docs/changelog/106526.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106526
-summary: Enhance search tier GC options
-area: Infra/CLI
-type: enhancement
-issues: []
diff --git a/docs/changelog/106531.yaml b/docs/changelog/106531.yaml
deleted file mode 100644
index 631d74185d2d8..0000000000000
--- a/docs/changelog/106531.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106531
-summary: Get and Query API Key with profile uid
-area: Security
-type: feature
-issues: []
diff --git a/docs/changelog/106563.yaml b/docs/changelog/106563.yaml
deleted file mode 100644
index 79476f909a04c..0000000000000
--- a/docs/changelog/106563.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106563
-summary: Improve short-circuiting downsample execution
-area: TSDB
-type: enhancement
-issues: []
diff --git a/docs/changelog/106575.yaml b/docs/changelog/106575.yaml
deleted file mode 100644
index fb5230a9edb3d..0000000000000
--- a/docs/changelog/106575.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106575
-summary: Unable to retrieve multiple stored field values
-area: "Search"
-type: bug
-issues: []
diff --git a/docs/changelog/106579.yaml b/docs/changelog/106579.yaml
deleted file mode 100644
index 104ed3066a6f6..0000000000000
--- a/docs/changelog/106579.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106579
-summary: "ESQL: Allow grouping key inside stats expressions"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106638.yaml b/docs/changelog/106638.yaml
deleted file mode 100644
index 019800bf03157..0000000000000
--- a/docs/changelog/106638.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106638
-summary: Allow users to get status of own async search tasks
-area: Authorization
-type: enhancement
-issues: []
diff --git a/docs/changelog/106654.yaml b/docs/changelog/106654.yaml
deleted file mode 100644
index 3443b68482443..0000000000000
--- a/docs/changelog/106654.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106654
-summary: "ES|QL: Fix usage of IN operator with TEXT fields"
-area: ES|QL
-type: bug
-issues:
- - 105379
diff --git a/docs/changelog/106685.yaml b/docs/changelog/106685.yaml
deleted file mode 100644
index ed4a16ba0666c..0000000000000
--- a/docs/changelog/106685.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106685
-summary: '`SharedBlobCacheService.maybeFetchRegion` should use `computeCacheFileRegionSize`'
-area: Snapshot/Restore
-type: bug
-issues: []
diff --git a/docs/changelog/106691.yaml b/docs/changelog/106691.yaml
deleted file mode 100644
index cbae9796e38c7..0000000000000
--- a/docs/changelog/106691.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106691
-summary: Fix range queries for float/half_float fields when bounds are out of type's
-  range
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/106708.yaml b/docs/changelog/106708.yaml
deleted file mode 100644
index b8fdd37e5f03f..0000000000000
--- a/docs/changelog/106708.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106708
-summary: Improve error message when rolling over DS alias
-area: Data streams
-type: bug
-issues:
- - 106137
diff --git a/docs/changelog/106714.yaml b/docs/changelog/106714.yaml
deleted file mode 100644
index 65b0acd77d764..0000000000000
--- a/docs/changelog/106714.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106714
-summary: Add non-indexed fields to ecs templates
-area: Data streams
-type: bug
-issues: []
diff --git a/docs/changelog/106720.yaml b/docs/changelog/106720.yaml
deleted file mode 100644
index 93358ed1d3dff..0000000000000
--- a/docs/changelog/106720.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106720
-summary: "ESQL: Fix treating all fields as MV in COUNT pushdown"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/106731.yaml b/docs/changelog/106731.yaml
deleted file mode 100644
index 0d8e16a8f9616..0000000000000
--- a/docs/changelog/106731.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106731
-summary: Fix field caps and field level security
-area: Security
-type: bug
-issues: []
diff --git a/docs/changelog/106745.yaml b/docs/changelog/106745.yaml
deleted file mode 100644
index a6cb035bd267a..0000000000000
--- a/docs/changelog/106745.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106745
-summary: Fix `AffixSetting.exists` to include secure settings
-area: Infra/Core
-type: bug
-issues: []
diff --git a/docs/changelog/106767.yaml b/docs/changelog/106767.yaml
deleted file mode 100644
index 8541e1b14f275..0000000000000
--- a/docs/changelog/106767.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106767
-summary: Handle pass-through subfields with deep nesting
-area: Mapping
-type: bug
-issues: []
diff --git a/docs/changelog/106796.yaml b/docs/changelog/106796.yaml
deleted file mode 100644
index 83eb99dba1603..0000000000000
--- a/docs/changelog/106796.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106796
-summary: Bulk loading enrich fields in ESQL
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106808.yaml b/docs/changelog/106808.yaml
deleted file mode 100644
index 287477fc302fd..0000000000000
--- a/docs/changelog/106808.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106808
-summary: Make OpenAI embeddings parser more flexible
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/106810.yaml b/docs/changelog/106810.yaml
deleted file mode 100644
index e93e5cf1e5361..0000000000000
--- a/docs/changelog/106810.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106810
-summary: "ES|QL: Improve support for TEXT fields in functions"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/106836.yaml b/docs/changelog/106836.yaml
deleted file mode 100644
index f561f44d9bb2d..0000000000000
--- a/docs/changelog/106836.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106836
-summary: Make int8_hnsw our default index for new dense-vector fields
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/106840.yaml b/docs/changelog/106840.yaml
deleted file mode 100644
index 3f6831e4907ca..0000000000000
--- a/docs/changelog/106840.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106840
-summary: Add total size in bytes to doc stats
-area: Stats
-type: enhancement
-issues:
- - 97670
diff --git a/docs/changelog/106851.yaml b/docs/changelog/106851.yaml
deleted file mode 100644
index 2ada6a6a4e088..0000000000000
--- a/docs/changelog/106851.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106851
-summary: Catching `StackOverflowErrors` from bad regexes in `GsubProcessor`
-area: Ingest Node
-type: bug
-issues: []
diff --git a/docs/changelog/106852.yaml b/docs/changelog/106852.yaml
deleted file mode 100644
index 2161b1ea22f30..0000000000000
--- a/docs/changelog/106852.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106852
-summary: Introduce ordinal bytesref block
-area: ES|QL
-type: enhancement
-issues:
- - 106387
diff --git a/docs/changelog/106860.yaml b/docs/changelog/106860.yaml
deleted file mode 100644
index 376f8753023b9..0000000000000
--- a/docs/changelog/106860.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106860
-summary: "[Profiling] Add TopN Functions API"
-area: Application
-type: enhancement
-issues: []
diff --git a/docs/changelog/106862.yaml b/docs/changelog/106862.yaml
deleted file mode 100644
index 3ca2660fc3f73..0000000000000
--- a/docs/changelog/106862.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106862
-summary: Extend support of `allowedFields` to `getMatchingFieldNames` and `getAllFields`
-area: "Mapping"
-type: bug
-issues: []
diff --git a/docs/changelog/106866.yaml b/docs/changelog/106866.yaml
deleted file mode 100644
index ffc34e5962850..0000000000000
--- a/docs/changelog/106866.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106866
-summary: Add ES|QL signum function
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106889.yaml b/docs/changelog/106889.yaml
deleted file mode 100644
index 7755081d09036..0000000000000
--- a/docs/changelog/106889.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106889
-summary: Slightly better geoip `databaseType` validation
-area: Ingest Node
-type: bug
-issues: []
diff --git a/docs/changelog/106899.yaml b/docs/changelog/106899.yaml
deleted file mode 100644
index a2db24236a47e..0000000000000
--- a/docs/changelog/106899.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106899
-summary: Add ES|QL Locate function
-area: ES|QL
-type: enhancement
-issues:
- - 106818
diff --git a/docs/changelog/106919.yaml b/docs/changelog/106919.yaml
deleted file mode 100644
index d8288095590de..0000000000000
--- a/docs/changelog/106919.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 106919
-summary: Fix downsample action request serialization
-area: Downsampling
-type: bug
-issues:
- - 106917
diff --git a/docs/changelog/106934.yaml b/docs/changelog/106934.yaml
deleted file mode 100644
index fbfce3118e8a6..0000000000000
--- a/docs/changelog/106934.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106934
-summary: Adjust array resizing in block builder
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/106952.yaml b/docs/changelog/106952.yaml
deleted file mode 100644
index 1b45bf6ca28a2..0000000000000
--- a/docs/changelog/106952.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 106952
-summary: Add Lucene spanish plural stemmer
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/106989.yaml b/docs/changelog/106989.yaml
deleted file mode 100644
index 47df5fe5b47d7..0000000000000
--- a/docs/changelog/106989.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-pr: 106989
-summary: Make force-stopping the transform always remove persistent task from cluster
-  state
-area: Transform
-type: bug
-issues:
- - 106811
diff --git a/docs/changelog/107007.yaml b/docs/changelog/107007.yaml
deleted file mode 100644
index b2a755171725b..0000000000000
--- a/docs/changelog/107007.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107007
-summary: "ESQL: Support ST_DISJOINT"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/107016.yaml b/docs/changelog/107016.yaml
deleted file mode 100644
index a2e32749a8008..0000000000000
--- a/docs/changelog/107016.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107016
-summary: "ESQL: Enable VALUES agg for datetime"
-area: Aggregations
-type: bug
-issues: []
diff --git a/docs/changelog/107038.yaml b/docs/changelog/107038.yaml
deleted file mode 100644
index e00b0d45a8a3a..0000000000000
--- a/docs/changelog/107038.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107038
-summary: Replace `UnsupportedOperationException` with `IllegalArgumentException` for non-existing columns
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/107041.yaml b/docs/changelog/107041.yaml
deleted file mode 100644
index b8b4f3d7c5690..0000000000000
--- a/docs/changelog/107041.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107041
-summary: '`DocumentParsingObserver` to accept an `indexName` to allow skipping system
-  indices'
-area: Infra/Metrics
-type: enhancement
-issues: []
diff --git a/docs/changelog/107046.yaml b/docs/changelog/107046.yaml
deleted file mode 100644
index 6c1373e09d17c..0000000000000
--- a/docs/changelog/107046.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107046
-summary: "[Security Solution] Add `read` permission for third party agent indices\
-  \ for `kibana_system`"
-area: Authorization
-type: enhancement
-issues: []
diff --git a/docs/changelog/107050.yaml b/docs/changelog/107050.yaml
deleted file mode 100644
index ecb375967ae44..0000000000000
--- a/docs/changelog/107050.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107050
-summary: Fix support for infinite `?master_timeout`
-area: Cluster Coordination
-type: bug
-issues: []
diff --git a/docs/changelog/107107.yaml b/docs/changelog/107107.yaml
deleted file mode 100644
index 5ca611befeb5d..0000000000000
--- a/docs/changelog/107107.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107107
-summary: Increase KDF iteration count in `KeyStoreWrapper`
-area: Infra/CLI
-type: enhancement
-issues: []
diff --git a/docs/changelog/107121.yaml b/docs/changelog/107121.yaml
deleted file mode 100644
index d46b1d58e9dfb..0000000000000
--- a/docs/changelog/107121.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107121
-summary: Add a flag to re-enable writes on the final index after an ILM shrink action.
-area: ILM+SLM
-type: enhancement
-issues:
-  - 106599
diff --git a/docs/changelog/107129.yaml b/docs/changelog/107129.yaml
deleted file mode 100644
index 6c9b9094962c1..0000000000000
--- a/docs/changelog/107129.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107129
-summary: Track ongoing search tasks
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/107158.yaml b/docs/changelog/107158.yaml
deleted file mode 100644
index 9589fe7e7264b..0000000000000
--- a/docs/changelog/107158.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107158
-summary: "ESQL: allow sorting by expressions and not only regular fields"
-area: ES|QL
-type: feature
-issues: []
diff --git a/docs/changelog/107178.yaml b/docs/changelog/107178.yaml
deleted file mode 100644
index 94a91357d38e6..0000000000000
--- a/docs/changelog/107178.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107178
-summary: "Add support for Azure OpenAI embeddings to inference service"
-area: Machine Learning
-type: feature
-issues: [ ]
diff --git a/docs/changelog/107183.yaml b/docs/changelog/107183.yaml
deleted file mode 100644
index 226d036456858..0000000000000
--- a/docs/changelog/107183.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107183
-summary: ES|QL fix no-length substring with supplementary (4-byte) character
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/107196.yaml b/docs/changelog/107196.yaml
deleted file mode 100644
index 9892ccf71856f..0000000000000
--- a/docs/changelog/107196.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107196
-summary: Add metric for calculating index flush time excluding waiting on locks
-area: Engine
-type: enhancement
-issues: []
diff --git a/docs/changelog/107224.yaml b/docs/changelog/107224.yaml
deleted file mode 100644
index b0d40c09b758a..0000000000000
--- a/docs/changelog/107224.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107224
-summary: "Enable 'encoder' and 'tags_schema' highlighting settings at field level"
-area: Highlighting
-type: enhancement
-issues:
-  - 94028
diff --git a/docs/changelog/107232.yaml b/docs/changelog/107232.yaml
deleted file mode 100644
index 1422848cb1c91..0000000000000
--- a/docs/changelog/107232.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107232
-summary: Only trigger action once per thread
-area: Transform
-type: bug
-issues:
- - 107215
diff --git a/docs/changelog/107242.yaml b/docs/changelog/107242.yaml
deleted file mode 100644
index 4a5e9821a1fa9..0000000000000
--- a/docs/changelog/107242.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107242
-summary: Added a timeout parameter to the inference API
-area: Machine Learning
-type: enhancement
-issues: [ ]
diff --git a/docs/changelog/107253.yaml b/docs/changelog/107253.yaml
deleted file mode 100644
index 6961b59231ea3..0000000000000
--- a/docs/changelog/107253.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107253
-summary: "[Connector API] Support cleaning up sync jobs when deleting a connector"
-area: Application
-type: feature
-issues: []
diff --git a/docs/changelog/107272.yaml b/docs/changelog/107272.yaml
deleted file mode 100644
index eb9e0c5e8bab8..0000000000000
--- a/docs/changelog/107272.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107272
-summary: "ESQL: extend BUCKET with spans"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/107287.yaml b/docs/changelog/107287.yaml
deleted file mode 100644
index 791f07fd1c729..0000000000000
--- a/docs/changelog/107287.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107287
-summary: Add support for the 'Anonymous IP' database to the geoip processor
-area: Ingest Node
-type: enhancement
-issues:
- - 90789
diff --git a/docs/changelog/107291.yaml b/docs/changelog/107291.yaml
deleted file mode 100644
index 3274fb77ef8c8..0000000000000
--- a/docs/changelog/107291.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107291
-summary: Support data streams in enrich policy indices
-area: Ingest Node
-type: enhancement
-issues:
- - 98836
diff --git a/docs/changelog/107303.yaml b/docs/changelog/107303.yaml
deleted file mode 100644
index 2e04ce6be3627..0000000000000
--- a/docs/changelog/107303.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107303
-summary: Create default word based chunker
-area: Machine Learning
-type: feature
-issues: []
diff --git a/docs/changelog/107312.yaml b/docs/changelog/107312.yaml
deleted file mode 100644
index 6ecd4179596e5..0000000000000
--- a/docs/changelog/107312.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107312
-summary: Fix NPE in ML assignment notifier
-area: Machine Learning
-type: bug
-issues: []
diff --git a/docs/changelog/107334.yaml b/docs/changelog/107334.yaml
deleted file mode 100644
index d1e8df2fa9c40..0000000000000
--- a/docs/changelog/107334.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107334
-summary: Adding `cache_stats` to geoip stats API
-area: Ingest Node
-type: enhancement
-issues: []
diff --git a/docs/changelog/107358.yaml b/docs/changelog/107358.yaml
deleted file mode 100644
index edb6deeffd100..0000000000000
--- a/docs/changelog/107358.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107358
-summary: Check node shutdown before fail
-area: Transform
-type: enhancement
-issues:
- - 100891
diff --git a/docs/changelog/107370.yaml b/docs/changelog/107370.yaml
deleted file mode 100644
index e7bdeef68cffe..0000000000000
--- a/docs/changelog/107370.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107370
-summary: Fork when handling remote field-caps responses
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/107377.yaml b/docs/changelog/107377.yaml
deleted file mode 100644
index a97f8b78dcce9..0000000000000
--- a/docs/changelog/107377.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-pr: 107377
-summary: Add support for the 'Enterprise' database to the geoip processor
-area: Ingest Node
-type: enhancement
-issues: []
-highlight:
-  title: "Preview: Support for the 'Anonymous IP' and 'Enterprise' databases in the geoip processor"
-  body: |-
-    As a Technical Preview, the {ref}/geoip-processor.html[`geoip`] processor can now use the commercial
-    https://www.maxmind.com/en/solutions/geoip2-enterprise-product-suite/enterprise-database[GeoIP2 'Enterprise']
-    and
-    https://www.maxmind.com/en/solutions/geoip2-enterprise-product-suite/anonymous-ip-database[GeoIP2 'Anonymous IP']
-    databases from MaxMind.
diff --git a/docs/changelog/107383.yaml b/docs/changelog/107383.yaml
deleted file mode 100644
index 07886ac96180c..0000000000000
--- a/docs/changelog/107383.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107383
-summary: Users with monitor privileges can access async_search/status endpoint 
-  even when setting keep_alive
-area: Authorization
-type: bug
-issues: []
diff --git a/docs/changelog/107411.yaml b/docs/changelog/107411.yaml
deleted file mode 100644
index fda040bcdab80..0000000000000
--- a/docs/changelog/107411.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107411
-summary: Invalidating cross cluster API keys requires `manage_security`
-area: Security
-type: enhancement
-issues: []
diff --git a/docs/changelog/107414.yaml b/docs/changelog/107414.yaml
deleted file mode 100644
index 60e31f22ca834..0000000000000
--- a/docs/changelog/107414.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-pr: 107414
-summary: "ESQL: median, count and `count_distinct` over constants"
-area: ES|QL
-type: bug
-issues:
- - 105248
- - 104900
diff --git a/docs/changelog/107447.yaml b/docs/changelog/107447.yaml
deleted file mode 100644
index 6ace513013e3e..0000000000000
--- a/docs/changelog/107447.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107447
-summary: "Fix regression in get index settings (human=true) where the version was not displayed in human-readable format"
-area: Infra/Core
-type: bug
-issues: []
diff --git a/docs/changelog/107449.yaml b/docs/changelog/107449.yaml
deleted file mode 100644
index 7f0b1bb826e94..0000000000000
--- a/docs/changelog/107449.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107449
-summary: Leverage ordinals in enrich lookup
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/107467.yaml b/docs/changelog/107467.yaml
deleted file mode 100644
index e775e5928770d..0000000000000
--- a/docs/changelog/107467.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107467
-summary: "[Connector API] Fix bug with filtering validation toXContent"
-area: Application
-type: bug
-issues: []
diff --git a/docs/changelog/107494.yaml b/docs/changelog/107494.yaml
deleted file mode 100644
index 1d71ce284a4a8..0000000000000
--- a/docs/changelog/107494.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107494
-summary: Handle infinity during synthetic source construction for scaled float field
-area: Mapping
-type: bug
-issues:
- - 107101
diff --git a/docs/changelog/107517.yaml b/docs/changelog/107517.yaml
deleted file mode 100644
index 4d7830699ad49..0000000000000
--- a/docs/changelog/107517.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107517
-summary: Add GET `_inference` for all inference endpoints
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/107533.yaml b/docs/changelog/107533.yaml
deleted file mode 100644
index da95cfd5b312e..0000000000000
--- a/docs/changelog/107533.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107533
-summary: Add setting for max connections to S3
-area: Snapshot/Restore
-type: enhancement
-issues: []
diff --git a/docs/changelog/107551.yaml b/docs/changelog/107551.yaml
deleted file mode 100644
index 78e64cc526638..0000000000000
--- a/docs/changelog/107551.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107551
-summary: Avoid attempting to load the same empty field twice in fetch phase
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/107577.yaml b/docs/changelog/107577.yaml
deleted file mode 100644
index a9a3c36a0e04d..0000000000000
--- a/docs/changelog/107577.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107577
-summary: "ESQL: Fix MV_DEDUPE when using data from an index"
-area: ES|QL
-type: bug
-issues:
- - 104745
diff --git a/docs/changelog/107578.yaml b/docs/changelog/107578.yaml
deleted file mode 100644
index 30746aeee6986..0000000000000
--- a/docs/changelog/107578.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107578
-summary: "ESQL: Allow reusing BUCKET grouping expressions in aggs"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/107598.yaml b/docs/changelog/107598.yaml
deleted file mode 100644
index 125bbe759d2ea..0000000000000
--- a/docs/changelog/107598.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107598
-summary: Fix bulk NPE when retrying failure redirect after cluster block
-area: Data streams
-type: bug
-issues: []
diff --git a/docs/changelog/107655.yaml b/docs/changelog/107655.yaml
deleted file mode 100644
index 7091224d211f1..0000000000000
--- a/docs/changelog/107655.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107655
-summary: "Use #addWithoutBreaking when adding a negative number of bytes to the circuit\
-  \ breaker in `SequenceMatcher`"
-area: EQL
-type: bug
-issues: []
diff --git a/docs/changelog/107678.yaml b/docs/changelog/107678.yaml
deleted file mode 100644
index 9be55dd4d6b96..0000000000000
--- a/docs/changelog/107678.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107678
-summary: Validate stats formatting in standard `InternalStats` constructor
-area: Aggregations
-type: bug
-issues:
- - 107671
diff --git a/docs/changelog/107743.yaml b/docs/changelog/107743.yaml
deleted file mode 100644
index fad45040330d2..0000000000000
--- a/docs/changelog/107743.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107743
-summary: Validate `model_id` is required when using the `learning_to_rank` rescorer
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/107828.yaml b/docs/changelog/107828.yaml
deleted file mode 100644
index ba0d44029203d..0000000000000
--- a/docs/changelog/107828.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107828
-summary: Update several references to `IndexVersion.toString` to use `toReleaseVersion`
-area: Infra/Core
-type: bug
-issues:
- - 107821
diff --git a/docs/changelog/107865.yaml b/docs/changelog/107865.yaml
deleted file mode 100644
index f7bb1d869eed5..0000000000000
--- a/docs/changelog/107865.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107865
-summary: Fix docs generation of signatures for variadic functions
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/107891.yaml b/docs/changelog/107891.yaml
deleted file mode 100644
index deb3fbd2258ff..0000000000000
--- a/docs/changelog/107891.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 107891
-summary: Fix `startOffset` must be non-negative error in XLMRoBERTa tokenizer
-area: Machine Learning
-type: bug
-issues:
- - 104626
diff --git a/docs/changelog/107902.yaml b/docs/changelog/107902.yaml
deleted file mode 100644
index 6b25f8c12df60..0000000000000
--- a/docs/changelog/107902.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107902
-summary: Update several references to `TransportVersion.toString` to use `toReleaseVersion`
-area: Infra/Core
-type: bug
-issues: []
diff --git a/docs/changelog/107969.yaml b/docs/changelog/107969.yaml
deleted file mode 100644
index ed63513d8d57d..0000000000000
--- a/docs/changelog/107969.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 107969
-summary: Disable PIT for remote clusters
-area: Transform
-type: bug
-issues: []
diff --git a/docs/changelog/108007.yaml b/docs/changelog/108007.yaml
deleted file mode 100644
index 5d24f8c87597c..0000000000000
--- a/docs/changelog/108007.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108007
-summary: Allow `typed_keys` for search application Search API
-area: Application
-type: feature
-issues: []
diff --git a/docs/changelog/108031.yaml b/docs/changelog/108031.yaml
deleted file mode 100644
index 0d02ddddbd472..0000000000000
--- a/docs/changelog/108031.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108031
-summary: Fix lingering license warning header
-area: License
-type: bug
-issues:
- - 107573
diff --git a/docs/changelog/108041.yaml b/docs/changelog/108041.yaml
deleted file mode 100644
index a82e0798dba21..0000000000000
--- a/docs/changelog/108041.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-pr: 108041
-summary: Handle parallel calls to `createWeight` when profiling is on
-area: Search
-type: bug
-issues:
- - 104131
- - 104235
diff --git a/docs/changelog/108101.yaml b/docs/changelog/108101.yaml
deleted file mode 100644
index e935ec1beecd6..0000000000000
--- a/docs/changelog/108101.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108101
-summary: "ESQL: Fix error message when failing to resolve aggregate groupings"
-area: ES|QL
-type: bug
-issues:
- - 108053
diff --git a/docs/changelog/108238.yaml b/docs/changelog/108238.yaml
deleted file mode 100644
index 607979c2eb0ac..0000000000000
--- a/docs/changelog/108238.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108238
-summary: "Nativeaccess: try to load all located libsystemds"
-area: Infra/Core
-type: bug
-issues:
- - 107878
diff --git a/docs/changelog/108257.yaml b/docs/changelog/108257.yaml
deleted file mode 100644
index ce2c72353af82..0000000000000
--- a/docs/changelog/108257.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108257
-summary: "ESQL: Log queries at debug level"
-area: ES|QL
-type: enhancement
-issues: []
diff --git a/docs/changelog/108365.yaml b/docs/changelog/108365.yaml
deleted file mode 100644
index d94486e2f3ea7..0000000000000
--- a/docs/changelog/108365.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108365
-summary: "[Bugfix] Connector API - fix status serialisation issue in termquery"
-area: Application
-type: bug
-issues: []
diff --git a/docs/changelog/108431.yaml b/docs/changelog/108431.yaml
deleted file mode 100644
index 84607b1b99ac3..0000000000000
--- a/docs/changelog/108431.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108431
-summary: "ESQL: Disable quoting in FROM command"
-area: ES|QL
-type: bug
-issues: []
diff --git a/docs/changelog/108518.yaml b/docs/changelog/108518.yaml
deleted file mode 100644
index aad823ccc89f6..0000000000000
--- a/docs/changelog/108518.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108518
-summary: Remove leading is_ prefix from Enterprise geoip docs
-area: Ingest Node
-type: bug
-issues: []
diff --git a/docs/changelog/108562.yaml b/docs/changelog/108562.yaml
deleted file mode 100644
index 2a0047fe807fd..0000000000000
--- a/docs/changelog/108562.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108562
-summary: Add `internalClusterTest` for and fix leak in `ExpandSearchPhase`
-area: Search
-type: bug
-issues:
- - 108369
diff --git a/docs/changelog/108571.yaml b/docs/changelog/108571.yaml
deleted file mode 100644
index b863ac90d9e5f..0000000000000
--- a/docs/changelog/108571.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108571
-summary: Workaround G1 bug for JDK 22 and 22.0.1
-area: Infra/CLI
-type: bug
-issues: []
diff --git a/docs/changelog/108600.yaml b/docs/changelog/108600.yaml
deleted file mode 100644
index 59177bf34114c..0000000000000
--- a/docs/changelog/108600.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-pr: 108600
-summary: "Prevent DLS/FLS if `replication` is assigned"
-area: Security
-type: breaking
-issues: [ ]
-breaking:
-  title: "Prevent DLS/FLS if `replication` is assigned"
-  area: REST API
-  details: For cross-cluster API keys, {es} no longer allows specifying document-level security (DLS)
-    or field-level security (FLS) in the `search` field, if `replication` is also specified.
-    {es} likewise blocks the use of any existing cross-cluster API keys that meet this condition.
-  impact: Remove any document-level security (DLS) or field-level security (FLS) definitions from the `search` field
-    for cross-cluster API keys that also have a `replication` field, or create two separate cross-cluster API keys,
-    one for search and one for replication.
-  notable: false
diff --git a/docs/changelog/108654.yaml b/docs/changelog/108654.yaml
deleted file mode 100644
index 9afae6a19ca80..0000000000000
--- a/docs/changelog/108654.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108654
-summary: Update bundled JDK to Java 22 (again)
-area: Packaging
-type: upgrade
-issues: []
diff --git a/docs/changelog/108736.yaml b/docs/changelog/108736.yaml
deleted file mode 100644
index 41e4084021e00..0000000000000
--- a/docs/changelog/108736.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108736
-summary: Harden field-caps request dispatcher
-area: Search
-type: bug
-issues: []
diff --git a/docs/changelog/108802.yaml b/docs/changelog/108802.yaml
deleted file mode 100644
index 7c28a81a1b353..0000000000000
--- a/docs/changelog/108802.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108802
-summary: Fix multithreading copies in lib vec
-area: Vector Search
-type: bug
-issues: []
diff --git a/docs/changelog/108834.yaml b/docs/changelog/108834.yaml
deleted file mode 100644
index 044056fa9a9da..0000000000000
--- a/docs/changelog/108834.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108834
-summary: "[ESQL] Mark `date_diff` as requiring all three arguments"
-area: ES|QL
-type: bug
-issues:
- - 108383
diff --git a/docs/changelog/108854.yaml b/docs/changelog/108854.yaml
deleted file mode 100644
index d6a880830f0d9..0000000000000
--- a/docs/changelog/108854.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 108854
-summary: "[Connector API] Fix bug with parsing *_doc_count nullable fields"
-area: Application
-type: bug
-issues: []
diff --git a/docs/changelog/108867.yaml b/docs/changelog/108867.yaml
deleted file mode 100644
index 545349dd84aeb..0000000000000
--- a/docs/changelog/108867.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108867
-summary: Fix for raw mapping merge of fields named "properties"
-area: Mapping
-type: bug
-issues:
- - 108866
diff --git a/docs/changelog/108900.yaml b/docs/changelog/108900.yaml
deleted file mode 100644
index 2a182f03ff8ce..0000000000000
--- a/docs/changelog/108900.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 108900
-summary: Treat 404 as empty register in `AzureBlobStore`
-area: Snapshot/Restore
-type: bug
-issues:
- - 108504
diff --git a/docs/changelog/109020.yaml b/docs/changelog/109020.yaml
deleted file mode 100644
index c3efb1a1409bf..0000000000000
--- a/docs/changelog/109020.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109020
-summary: Only skip deleting a downsampled index if downsampling is in progress as
-  part of DSL retention
-area: Data streams
-type: bug
-issues: []
diff --git a/docs/changelog/109034.yaml b/docs/changelog/109034.yaml
deleted file mode 100644
index cdf1f6fe28d8d..0000000000000
--- a/docs/changelog/109034.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109034
-summary: Fix IOOBE in TTest aggregation when using filters
-area: Aggregations
-type: bug
-issues: []
diff --git a/docs/changelog/109048.yaml b/docs/changelog/109048.yaml
deleted file mode 100644
index 8bae082404ecd..0000000000000
--- a/docs/changelog/109048.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109048
-summary: Guard against a null scorer in painless execute
-area: Infra/Scripting
-type: bug
-issues:
- - 43541
diff --git a/docs/changelog/109097.yaml b/docs/changelog/109097.yaml
deleted file mode 100644
index a7520f4eaa9be..0000000000000
--- a/docs/changelog/109097.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109097
-summary: "[Connector API] Fix bug with with wrong target index for access control\
-  \ sync"
-area: Application
-type: bug
-issues: []
diff --git a/docs/changelog/109148.yaml b/docs/changelog/109148.yaml
deleted file mode 100644
index 902da6f1a1db3..0000000000000
--- a/docs/changelog/109148.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 109148
-summary: Fix double-pausing shard snapshot
-area: Snapshot/Restore
-type: bug
-issues:
- - 109143
diff --git a/docs/changelog/109173.yaml b/docs/changelog/109173.yaml
deleted file mode 100644
index 9f4f73a6f74c8..0000000000000
--- a/docs/changelog/109173.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 109173
-summary: Wrap "Pattern too complex" exception into an `IllegalArgumentException`
-area: Mapping
-type: bug
-issues: []
diff --git a/docs/changelog/97072.yaml b/docs/changelog/97072.yaml
deleted file mode 100644
index 686b30952b646..0000000000000
--- a/docs/changelog/97072.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 97072
-summary: Log when update AffixSetting using addAffixMapUpdateConsumer
-area: Infra/Logging
-type: bug
-issues: []
diff --git a/docs/changelog/97561.yaml b/docs/changelog/97561.yaml
deleted file mode 100644
index cacefbf7e4ca3..0000000000000
--- a/docs/changelog/97561.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 97561
-summary: Add index forecasts to /_cat/allocation output
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/99048.yaml b/docs/changelog/99048.yaml
deleted file mode 100644
index 722c145dae78f..0000000000000
--- a/docs/changelog/99048.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 99048
-summary: String sha512() painless function
-area: Infra/Scripting
-type: enhancement
-issues:
-  - 97691

From b1c798237879dab3261f4c31f365e16024e2cd1e Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Wed, 5 Jun 2024 11:14:03 -0400
Subject: [PATCH 170/208] [ESQL] Migrate Optimizer rules and associated tests
 (#109216)

This PR tries to reconcile the tests for the logical plan optimizer rules. I deleted tests and rules in esql-core that had already been pulled into esql, and pulled in most of the remaining tests, and several of the rules.

Already migrated and removed from core:
- CombineDisjunction
- InferIsNotNull
- PropagateEquals

Newly migrated rules:
- ReplaceRegexMatch
- BinaryComparisonSimplification
- CombineBinaryComparisons

This enables removing the (hopefully) unused operator base classes (Add, Equals, etc) in core, or at least removes one blocker to that. There may be more tests in core that reference these classes and need to be migrated, I haven't exhaustively looked yet.
---
 .../esql/core/optimizer/OptimizerRules.java   | 1049 +----------
 .../core/optimizer/OptimizerRulesTests.java   | 1657 -----------------
 .../esql/optimizer/LogicalPlanOptimizer.java  |   25 +-
 .../xpack/esql/optimizer/OptimizerRules.java  |    6 -
 .../esql/optimizer/OptimizerRulesTests.java   |  383 ++++
 5 files changed, 409 insertions(+), 2711 deletions(-)

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java
index 7759b62d3d187..137f440f03b7e 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRules.java
@@ -8,8 +8,6 @@
 
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
-import org.elasticsearch.xpack.esql.core.expression.Attribute;
-import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
@@ -22,7 +20,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
-import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
@@ -31,27 +28,18 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Equals;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.GreaterThan;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.GreaterThanOrEqual;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.In;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.LessThan;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.NotEquals;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.NullEquals;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.StringPattern;
 import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.core.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.rule.Rule;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.ReflectionUtils;
 
 import java.time.ZoneId;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.LinkedList;
@@ -60,7 +48,6 @@
 import java.util.Set;
 import java.util.function.BiFunction;
 
-import static java.util.Collections.emptySet;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.expression.predicate.Predicates.combineAnd;
@@ -248,43 +235,6 @@ protected Expression maybeSimplifyNegatable(Expression e) {
         }
     }
 
-    public static class BinaryComparisonSimplification extends OptimizerExpressionRule<BinaryComparison> {
-
-        public BinaryComparisonSimplification() {
-            super(TransformDirection.DOWN);
-        }
-
-        @Override
-        protected Expression rule(BinaryComparison bc) {
-            Expression l = bc.left();
-            Expression r = bc.right();
-
-            // true for equality
-            if (bc instanceof Equals || bc instanceof GreaterThanOrEqual || bc instanceof LessThanOrEqual) {
-                if (l.nullable() == Nullability.FALSE && r.nullable() == Nullability.FALSE && l.semanticEquals(r)) {
-                    return new Literal(bc.source(), Boolean.TRUE, DataType.BOOLEAN);
-                }
-            }
-            if (bc instanceof NullEquals) {
-                if (l.semanticEquals(r)) {
-                    return new Literal(bc.source(), Boolean.TRUE, DataType.BOOLEAN);
-                }
-                if (Expressions.isNull(r)) {
-                    return new IsNull(bc.source(), l);
-                }
-            }
-
-            // false for equality
-            if (bc instanceof NotEquals || bc instanceof GreaterThan || bc instanceof LessThan) {
-                if (l.nullable() == Nullability.FALSE && r.nullable() == Nullability.FALSE && l.semanticEquals(r)) {
-                    return new Literal(bc.source(), Boolean.FALSE, DataType.BOOLEAN);
-                }
-            }
-
-            return bc;
-        }
-    }
-
     public static final class LiteralsOnTheRight extends OptimizerExpressionRule<BinaryOperator<?, ?, ?, ?>> {
 
         public LiteralsOnTheRight() {
@@ -297,887 +247,6 @@ public LiteralsOnTheRight() {
         }
     }
 
-    /**
-     * Propagate Equals to eliminate conjuncted Ranges or BinaryComparisons.
-     * When encountering a different Equals, non-containing {@link Range} or {@link BinaryComparison}, the conjunction becomes false.
-     * When encountering a containing {@link Range}, {@link BinaryComparison} or {@link NotEquals}, these get eliminated by the equality.
-     *
-     * Since this rule can eliminate Ranges and BinaryComparisons, it should be applied before {@link CombineBinaryComparisons}.
-     *
-     * This rule doesn't perform any promotion of {@link BinaryComparison}s, that is handled by
-     * {@link CombineBinaryComparisons} on purpose as the resulting Range might be foldable
-     * (which is picked by the folding rule on the next run).
-     */
-    public static final class PropagateEquals extends OptimizerExpressionRule<BinaryLogic> {
-
-        public PropagateEquals() {
-            super(TransformDirection.DOWN);
-        }
-
-        @Override
-        public Expression rule(BinaryLogic e) {
-            if (e instanceof And) {
-                return propagate((And) e);
-            } else if (e instanceof Or) {
-                return propagate((Or) e);
-            }
-            return e;
-        }
-
-        // combine conjunction
-        private static Expression propagate(And and) {
-            List<Range> ranges = new ArrayList<>();
-            // Only equalities, not-equalities and inequalities with a foldable .right are extracted separately;
-            // the others go into the general 'exps'.
-            List<BinaryComparison> equals = new ArrayList<>();
-            List<NotEquals> notEquals = new ArrayList<>();
-            List<BinaryComparison> inequalities = new ArrayList<>();
-            List<Expression> exps = new ArrayList<>();
-
-            boolean changed = false;
-
-            for (Expression ex : Predicates.splitAnd(and)) {
-                if (ex instanceof Range) {
-                    ranges.add((Range) ex);
-                } else if (ex instanceof Equals || ex instanceof NullEquals) {
-                    BinaryComparison otherEq = (BinaryComparison) ex;
-                    // equals on different values evaluate to FALSE
-                    // ignore date/time fields as equality comparison might actually be a range check
-                    if (otherEq.right().foldable() && DataType.isDateTime(otherEq.left().dataType()) == false) {
-                        for (BinaryComparison eq : equals) {
-                            if (otherEq.left().semanticEquals(eq.left())) {
-                                Integer comp = BinaryComparison.compare(eq.right().fold(), otherEq.right().fold());
-                                if (comp != null) {
-                                    // var cannot be equal to two different values at the same time
-                                    if (comp != 0) {
-                                        return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
-                                    }
-                                }
-                            }
-                        }
-                        equals.add(otherEq);
-                    } else {
-                        exps.add(otherEq);
-                    }
-                } else if (ex instanceof GreaterThan
-                    || ex instanceof GreaterThanOrEqual
-                    || ex instanceof LessThan
-                    || ex instanceof LessThanOrEqual) {
-                        BinaryComparison bc = (BinaryComparison) ex;
-                        if (bc.right().foldable()) {
-                            inequalities.add(bc);
-                        } else {
-                            exps.add(ex);
-                        }
-                    } else if (ex instanceof NotEquals otherNotEq) {
-                        if (otherNotEq.right().foldable()) {
-                            notEquals.add(otherNotEq);
-                        } else {
-                            exps.add(ex);
-                        }
-                    } else {
-                        exps.add(ex);
-                    }
-            }
-
-            // check
-            for (BinaryComparison eq : equals) {
-                Object eqValue = eq.right().fold();
-
-                for (Iterator<Range> iterator = ranges.iterator(); iterator.hasNext();) {
-                    Range range = iterator.next();
-
-                    if (range.value().semanticEquals(eq.left())) {
-                        // if equals is outside the interval, evaluate the whole expression to FALSE
-                        if (range.lower().foldable()) {
-                            Integer compare = BinaryComparison.compare(range.lower().fold(), eqValue);
-                            if (compare != null && (
-                            // eq outside the lower boundary
-                            compare > 0 ||
-                            // eq matches the boundary but should not be included
-                                (compare == 0 && range.includeLower() == false))) {
-                                return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
-                            }
-                        }
-                        if (range.upper().foldable()) {
-                            Integer compare = BinaryComparison.compare(range.upper().fold(), eqValue);
-                            if (compare != null && (
-                            // eq outside the upper boundary
-                            compare < 0 ||
-                            // eq matches the boundary but should not be included
-                                (compare == 0 && range.includeUpper() == false))) {
-                                return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
-                            }
-                        }
-
-                        // it's in the range and thus, remove it
-                        iterator.remove();
-                        changed = true;
-                    }
-                }
-
-                // evaluate all NotEquals against the Equal
-                for (Iterator<NotEquals> iter = notEquals.iterator(); iter.hasNext();) {
-                    NotEquals neq = iter.next();
-                    if (eq.left().semanticEquals(neq.left())) {
-                        Integer comp = BinaryComparison.compare(eqValue, neq.right().fold());
-                        if (comp != null) {
-                            if (comp == 0) { // clashing and conflicting: a = 1 AND a != 1
-                                return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
-                            } else { // clashing and redundant: a = 1 AND a != 2
-                                iter.remove();
-                                changed = true;
-                            }
-                        }
-                    }
-                }
-
-                // evaluate all inequalities against the Equal
-                for (Iterator<BinaryComparison> iter = inequalities.iterator(); iter.hasNext();) {
-                    BinaryComparison bc = iter.next();
-                    if (eq.left().semanticEquals(bc.left())) {
-                        Integer compare = BinaryComparison.compare(eqValue, bc.right().fold());
-                        if (compare != null) {
-                            if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { // a = 2 AND a </<= ?
-                                if ((compare == 0 && bc instanceof LessThan) || // a = 2 AND a < 2
-                                    0 < compare) { // a = 2 AND a </<= 1
-                                    return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
-                                }
-                            } else if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) { // a = 2 AND a >/>= ?
-                                if ((compare == 0 && bc instanceof GreaterThan) || // a = 2 AND a > 2
-                                    compare < 0) { // a = 2 AND a >/>= 3
-                                    return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN);
-                                }
-                            }
-
-                            iter.remove();
-                            changed = true;
-                        }
-                    }
-                }
-            }
-
-            return changed ? Predicates.combineAnd(CollectionUtils.combine(exps, equals, notEquals, inequalities, ranges)) : and;
-        }
-
-        // combine disjunction:
-        // a = 2 OR a > 3 -> nop; a = 2 OR a > 1 -> a > 1
-        // a = 2 OR a < 3 -> a < 3; a = 2 OR a < 1 -> nop
-        // a = 2 OR 3 < a < 5 -> nop; a = 2 OR 1 < a < 3 -> 1 < a < 3; a = 2 OR 0 < a < 1 -> nop
-        // a = 2 OR a != 2 -> TRUE; a = 2 OR a = 5 -> nop; a = 2 OR a != 5 -> a != 5
-        private static Expression propagate(Or or) {
-            List<Expression> exps = new ArrayList<>();
-            List<Equals> equals = new ArrayList<>(); // foldable right term Equals
-            List<NotEquals> notEquals = new ArrayList<>(); // foldable right term NotEquals
-            List<Range> ranges = new ArrayList<>();
-            List<BinaryComparison> inequalities = new ArrayList<>(); // foldable right term (=limit) BinaryComparision
-
-            // split expressions by type
-            for (Expression ex : Predicates.splitOr(or)) {
-                if (ex instanceof Equals eq) {
-                    if (eq.right().foldable()) {
-                        equals.add(eq);
-                    } else {
-                        exps.add(ex);
-                    }
-                } else if (ex instanceof NotEquals neq) {
-                    if (neq.right().foldable()) {
-                        notEquals.add(neq);
-                    } else {
-                        exps.add(ex);
-                    }
-                } else if (ex instanceof Range) {
-                    ranges.add((Range) ex);
-                } else if (ex instanceof BinaryComparison bc) {
-                    if (bc.right().foldable()) {
-                        inequalities.add(bc);
-                    } else {
-                        exps.add(ex);
-                    }
-                } else {
-                    exps.add(ex);
-                }
-            }
-
-            boolean updated = false; // has the expression been modified?
-
-            // evaluate the impact of each Equal over the different types of Expressions
-            for (Iterator<Equals> iterEq = equals.iterator(); iterEq.hasNext();) {
-                Equals eq = iterEq.next();
-                Object eqValue = eq.right().fold();
-                boolean removeEquals = false;
-
-                // Equals OR NotEquals
-                for (NotEquals neq : notEquals) {
-                    if (eq.left().semanticEquals(neq.left())) { // a = 2 OR a != ? -> ...
-                        Integer comp = BinaryComparison.compare(eqValue, neq.right().fold());
-                        if (comp != null) {
-                            if (comp == 0) { // a = 2 OR a != 2 -> TRUE
-                                return TRUE;
-                            } else { // a = 2 OR a != 5 -> a != 5
-                                removeEquals = true;
-                                break;
-                            }
-                        }
-                    }
-                }
-                if (removeEquals) {
-                    iterEq.remove();
-                    updated = true;
-                    continue;
-                }
-
-                // Equals OR Range
-                for (int i = 0; i < ranges.size(); i++) { // might modify list, so use index loop
-                    Range range = ranges.get(i);
-                    if (eq.left().semanticEquals(range.value())) {
-                        Integer lowerComp = range.lower().foldable() ? BinaryComparison.compare(eqValue, range.lower().fold()) : null;
-                        Integer upperComp = range.upper().foldable() ? BinaryComparison.compare(eqValue, range.upper().fold()) : null;
-
-                        if (lowerComp != null && lowerComp == 0) {
-                            if (range.includeLower() == false) { // a = 2 OR 2 < a < ? -> 2 <= a < ?
-                                ranges.set(
-                                    i,
-                                    new Range(
-                                        range.source(),
-                                        range.value(),
-                                        range.lower(),
-                                        true,
-                                        range.upper(),
-                                        range.includeUpper(),
-                                        range.zoneId()
-                                    )
-                                );
-                            } // else : a = 2 OR 2 <= a < ? -> 2 <= a < ?
-                            removeEquals = true; // update range with lower equality instead or simply superfluous
-                            break;
-                        } else if (upperComp != null && upperComp == 0) {
-                            if (range.includeUpper() == false) { // a = 2 OR ? < a < 2 -> ? < a <= 2
-                                ranges.set(
-                                    i,
-                                    new Range(
-                                        range.source(),
-                                        range.value(),
-                                        range.lower(),
-                                        range.includeLower(),
-                                        range.upper(),
-                                        true,
-                                        range.zoneId()
-                                    )
-                                );
-                            } // else : a = 2 OR ? < a <= 2 -> ? < a <= 2
-                            removeEquals = true; // update range with upper equality instead
-                            break;
-                        } else if (lowerComp != null && upperComp != null) {
-                            if (0 < lowerComp && upperComp < 0) { // a = 2 OR 1 < a < 3
-                                removeEquals = true; // equality is superfluous
-                                break;
-                            }
-                        }
-                    }
-                }
-                if (removeEquals) {
-                    iterEq.remove();
-                    updated = true;
-                    continue;
-                }
-
-                // Equals OR Inequality
-                for (int i = 0; i < inequalities.size(); i++) {
-                    BinaryComparison bc = inequalities.get(i);
-                    if (eq.left().semanticEquals(bc.left())) {
-                        Integer comp = BinaryComparison.compare(eqValue, bc.right().fold());
-                        if (comp != null) {
-                            if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) {
-                                if (comp < 0) { // a = 1 OR a > 2 -> nop
-                                    continue;
-                                } else if (comp == 0 && bc instanceof GreaterThan) { // a = 2 OR a > 2 -> a >= 2
-                                    inequalities.set(i, new GreaterThanOrEqual(bc.source(), bc.left(), bc.right(), bc.zoneId()));
-                                } // else (0 < comp || bc instanceof GreaterThanOrEqual) :
-                                  // a = 3 OR a > 2 -> a > 2; a = 2 OR a => 2 -> a => 2
-
-                                removeEquals = true; // update range with equality instead or simply superfluous
-                                break;
-                            } else if (bc instanceof LessThan || bc instanceof LessThanOrEqual) {
-                                if (comp > 0) { // a = 2 OR a < 1 -> nop
-                                    continue;
-                                }
-                                if (comp == 0 && bc instanceof LessThan) { // a = 2 OR a < 2 -> a <= 2
-                                    inequalities.set(i, new LessThanOrEqual(bc.source(), bc.left(), bc.right(), bc.zoneId()));
-                                } // else (comp < 0 || bc instanceof LessThanOrEqual) : a = 2 OR a < 3 -> a < 3; a = 2 OR a <= 2 -> a <= 2
-                                removeEquals = true; // update range with equality instead or simply superfluous
-                                break;
-                            }
-                        }
-                    }
-                }
-                if (removeEquals) {
-                    iterEq.remove();
-                    updated = true;
-                }
-            }
-
-            return updated ? Predicates.combineOr(CollectionUtils.combine(exps, equals, notEquals, inequalities, ranges)) : or;
-        }
-    }
-
-    public static final class CombineBinaryComparisons extends OptimizerExpressionRule<BinaryLogic> {
-
-        public CombineBinaryComparisons() {
-            super(TransformDirection.DOWN);
-        }
-
-        @Override
-        public Expression rule(BinaryLogic e) {
-            if (e instanceof And) {
-                return combine((And) e);
-            } else if (e instanceof Or) {
-                return combine((Or) e);
-            }
-            return e;
-        }
-
-        // combine conjunction
-        private static Expression combine(And and) {
-            List<Range> ranges = new ArrayList<>();
-            List<BinaryComparison> bcs = new ArrayList<>();
-            List<Expression> exps = new ArrayList<>();
-
-            boolean changed = false;
-
-            List<Expression> andExps = Predicates.splitAnd(and);
-            // Ranges need to show up before BinaryComparisons in list, to allow the latter be optimized away into a Range, if possible.
-            // NotEquals need to be last in list, to have a complete set of Ranges (ranges) and BinaryComparisons (bcs) and allow these to
-            // optimize the NotEquals away.
-            andExps.sort((o1, o2) -> {
-                if (o1 instanceof Range && o2 instanceof Range) {
-                    return 0; // keep ranges' order
-                } else if (o1 instanceof Range || o2 instanceof Range) {
-                    return o2 instanceof Range ? 1 : -1; // push Ranges down
-                } else if (o1 instanceof NotEquals && o2 instanceof NotEquals) {
-                    return 0; // keep NotEquals' order
-                } else if (o1 instanceof NotEquals || o2 instanceof NotEquals) {
-                    return o1 instanceof NotEquals ? 1 : -1; // push NotEquals up
-                } else {
-                    return 0; // keep non-Ranges' and non-NotEquals' order
-                }
-            });
-            for (Expression ex : andExps) {
-                if (ex instanceof Range r) {
-                    if (findExistingRange(r, ranges, true)) {
-                        changed = true;
-                    } else {
-                        ranges.add(r);
-                    }
-                } else if (ex instanceof BinaryComparison bc && (ex instanceof Equals || ex instanceof NotEquals) == false) {
-
-                    if (bc.right().foldable() && (findConjunctiveComparisonInRange(bc, ranges) || findExistingComparison(bc, bcs, true))) {
-                        changed = true;
-                    } else {
-                        bcs.add(bc);
-                    }
-                } else if (ex instanceof NotEquals neq) {
-                    if (neq.right().foldable() && notEqualsIsRemovableFromConjunction(neq, ranges, bcs)) {
-                        // the non-equality can simply be dropped: either superfluous or has been merged with an updated range/inequality
-                        changed = true;
-                    } else { // not foldable OR not overlapping
-                        exps.add(ex);
-                    }
-                } else {
-                    exps.add(ex);
-                }
-            }
-
-            // finally try combining any left BinaryComparisons into possible Ranges
-            // this could be a different rule but it's clearer here wrt the order of comparisons
-
-            for (int i = 0, step = 1; i < bcs.size() - 1; i += step, step = 1) {
-                BinaryComparison main = bcs.get(i);
-
-                for (int j = i + 1; j < bcs.size(); j++) {
-                    BinaryComparison other = bcs.get(j);
-
-                    if (main.left().semanticEquals(other.left())) {
-                        // >/>= AND </<=
-                        if ((main instanceof GreaterThan || main instanceof GreaterThanOrEqual)
-                            && (other instanceof LessThan || other instanceof LessThanOrEqual)) {
-                            bcs.remove(j);
-                            bcs.remove(i);
-
-                            ranges.add(
-                                new Range(
-                                    and.source(),
-                                    main.left(),
-                                    main.right(),
-                                    main instanceof GreaterThanOrEqual,
-                                    other.right(),
-                                    other instanceof LessThanOrEqual,
-                                    main.zoneId()
-                                )
-                            );
-
-                            changed = true;
-                            step = 0;
-                            break;
-                        }
-                        // </<= AND >/>=
-                        else if ((other instanceof GreaterThan || other instanceof GreaterThanOrEqual)
-                            && (main instanceof LessThan || main instanceof LessThanOrEqual)) {
-                                bcs.remove(j);
-                                bcs.remove(i);
-
-                                ranges.add(
-                                    new Range(
-                                        and.source(),
-                                        main.left(),
-                                        other.right(),
-                                        other instanceof GreaterThanOrEqual,
-                                        main.right(),
-                                        main instanceof LessThanOrEqual,
-                                        main.zoneId()
-                                    )
-                                );
-
-                                changed = true;
-                                step = 0;
-                                break;
-                            }
-                    }
-                }
-            }
-
-            return changed ? Predicates.combineAnd(CollectionUtils.combine(exps, bcs, ranges)) : and;
-        }
-
-        // combine disjunction
-        private static Expression combine(Or or) {
-            List<BinaryComparison> bcs = new ArrayList<>();
-            List<Range> ranges = new ArrayList<>();
-            List<Expression> exps = new ArrayList<>();
-
-            boolean changed = false;
-
-            for (Expression ex : Predicates.splitOr(or)) {
-                if (ex instanceof Range r) {
-                    if (findExistingRange(r, ranges, false)) {
-                        changed = true;
-                    } else {
-                        ranges.add(r);
-                    }
-                } else if (ex instanceof BinaryComparison bc) {
-                    if (bc.right().foldable() && findExistingComparison(bc, bcs, false)) {
-                        changed = true;
-                    } else {
-                        bcs.add(bc);
-                    }
-                } else {
-                    exps.add(ex);
-                }
-            }
-
-            return changed ? Predicates.combineOr(CollectionUtils.combine(exps, bcs, ranges)) : or;
-        }
-
-        private static boolean findExistingRange(Range main, List<Range> ranges, boolean conjunctive) {
-            if (main.lower().foldable() == false && main.upper().foldable() == false) {
-                return false;
-            }
-            // NB: the loop modifies the list (hence why the int is used)
-            for (int i = 0; i < ranges.size(); i++) {
-                Range other = ranges.get(i);
-
-                if (main.value().semanticEquals(other.value())) {
-
-                    // make sure the comparison was done
-                    boolean compared = false;
-
-                    boolean lower = false;
-                    boolean upper = false;
-                    // boundary equality (useful to differentiate whether a range is included or not)
-                    // and thus whether it should be preserved or ignored
-                    boolean lowerEq = false;
-                    boolean upperEq = false;
-
-                    // evaluate lower
-                    if (main.lower().foldable() && other.lower().foldable()) {
-                        compared = true;
-
-                        Integer comp = BinaryComparison.compare(main.lower().fold(), other.lower().fold());
-                        // values are comparable
-                        if (comp != null) {
-                            // boundary equality
-                            lowerEq = comp == 0 && main.includeLower() == other.includeLower();
-                            // AND
-                            if (conjunctive) {
-                                // (2 < a < 3) AND (1 < a < 3) -> (2 < a < 3)
-                                lower = comp > 0 ||
-                                // (2 < a < 3) AND (2 <= a < 3) -> (2 < a < 3)
-                                    (comp == 0 && main.includeLower() == false && other.includeLower());
-                            }
-                            // OR
-                            else {
-                                // (1 < a < 3) OR (2 < a < 3) -> (1 < a < 3)
-                                lower = comp < 0 ||
-                                // (2 <= a < 3) OR (2 < a < 3) -> (2 <= a < 3)
-                                    (comp == 0 && main.includeLower() && other.includeLower() == false) || lowerEq;
-                            }
-                        }
-                    }
-                    // evaluate upper
-                    if (main.upper().foldable() && other.upper().foldable()) {
-                        compared = true;
-
-                        Integer comp = BinaryComparison.compare(main.upper().fold(), other.upper().fold());
-                        // values are comparable
-                        if (comp != null) {
-                            // boundary equality
-                            upperEq = comp == 0 && main.includeUpper() == other.includeUpper();
-
-                            // AND
-                            if (conjunctive) {
-                                // (1 < a < 2) AND (1 < a < 3) -> (1 < a < 2)
-                                upper = comp < 0 ||
-                                // (1 < a < 2) AND (1 < a <= 2) -> (1 < a < 2)
-                                    (comp == 0 && main.includeUpper() == false && other.includeUpper());
-                            }
-                            // OR
-                            else {
-                                // (1 < a < 3) OR (1 < a < 2) -> (1 < a < 3)
-                                upper = comp > 0 ||
-                                // (1 < a <= 3) OR (1 < a < 3) -> (2 < a < 3)
-                                    (comp == 0 && main.includeUpper() && other.includeUpper() == false) || upperEq;
-                            }
-                        }
-                    }
-
-                    // AND - at least one of lower or upper
-                    if (conjunctive) {
-                        // can tighten range
-                        if (lower || upper) {
-                            ranges.set(
-                                i,
-                                new Range(
-                                    main.source(),
-                                    main.value(),
-                                    lower ? main.lower() : other.lower(),
-                                    lower ? main.includeLower() : other.includeLower(),
-                                    upper ? main.upper() : other.upper(),
-                                    upper ? main.includeUpper() : other.includeUpper(),
-                                    main.zoneId()
-                                )
-                            );
-                        }
-
-                        // range was comparable
-                        return compared;
-                    }
-                    // OR - needs both upper and lower to loosen range
-                    else {
-                        // can loosen range
-                        if (lower && upper) {
-                            ranges.set(
-                                i,
-                                new Range(
-                                    main.source(),
-                                    main.value(),
-                                    main.lower(),
-                                    main.includeLower(),
-                                    main.upper(),
-                                    main.includeUpper(),
-                                    main.zoneId()
-                                )
-                            );
-                            return true;
-                        }
-
-                        // if the range in included, no need to add it
-                        return compared && (((lower && lowerEq == false) || (upper && upperEq == false)) == false);
-                    }
-                }
-            }
-            return false;
-        }
-
-        private static boolean findConjunctiveComparisonInRange(BinaryComparison main, List<Range> ranges) {
-            Object value = main.right().fold();
-
-            // NB: the loop modifies the list (hence why the int is used)
-            for (int i = 0; i < ranges.size(); i++) {
-                Range other = ranges.get(i);
-
-                if (main.left().semanticEquals(other.value())) {
-
-                    if (main instanceof GreaterThan || main instanceof GreaterThanOrEqual) {
-                        if (other.lower().foldable()) {
-                            Integer comp = BinaryComparison.compare(value, other.lower().fold());
-                            if (comp != null) {
-                                // 2 < a AND (2 <= a < 3) -> 2 < a < 3
-                                boolean lowerEq = comp == 0 && other.includeLower() && main instanceof GreaterThan;
-                                // 2 < a AND (1 < a < 3) -> 2 < a < 3
-                                boolean lower = comp > 0 || lowerEq;
-
-                                if (lower) {
-                                    ranges.set(
-                                        i,
-                                        new Range(
-                                            other.source(),
-                                            other.value(),
-                                            main.right(),
-                                            lowerEq ? false : main instanceof GreaterThanOrEqual,
-                                            other.upper(),
-                                            other.includeUpper(),
-                                            other.zoneId()
-                                        )
-                                    );
-                                }
-
-                                // found a match
-                                return true;
-                            }
-                        }
-                    } else if (main instanceof LessThan || main instanceof LessThanOrEqual) {
-                        if (other.upper().foldable()) {
-                            Integer comp = BinaryComparison.compare(value, other.upper().fold());
-                            if (comp != null) {
-                                // a < 2 AND (1 < a <= 2) -> 1 < a < 2
-                                boolean upperEq = comp == 0 && other.includeUpper() && main instanceof LessThan;
-                                // a < 2 AND (1 < a < 3) -> 1 < a < 2
-                                boolean upper = comp < 0 || upperEq;
-
-                                if (upper) {
-                                    ranges.set(
-                                        i,
-                                        new Range(
-                                            other.source(),
-                                            other.value(),
-                                            other.lower(),
-                                            other.includeLower(),
-                                            main.right(),
-                                            upperEq ? false : main instanceof LessThanOrEqual,
-                                            other.zoneId()
-                                        )
-                                    );
-                                }
-
-                                // found a match
-                                return true;
-                            }
-                        }
-                    }
-
-                    return false;
-                }
-            }
-            return false;
-        }
-
-        /**
-         * Find commonalities between the given comparison in the given list.
-         * The method can be applied both for conjunctive (AND) or disjunctive purposes (OR).
-         */
-        private static boolean findExistingComparison(BinaryComparison main, List<BinaryComparison> bcs, boolean conjunctive) {
-            Object value = main.right().fold();
-
-            // NB: the loop modifies the list (hence why the int is used)
-            for (int i = 0; i < bcs.size(); i++) {
-                BinaryComparison other = bcs.get(i);
-                // skip if cannot evaluate
-                if (other.right().foldable() == false) {
-                    continue;
-                }
-                // if bc is a higher/lower value or gte vs gt, use it instead
-                if ((other instanceof GreaterThan || other instanceof GreaterThanOrEqual)
-                    && (main instanceof GreaterThan || main instanceof GreaterThanOrEqual)) {
-
-                    if (main.left().semanticEquals(other.left())) {
-                        Integer compare = BinaryComparison.compare(value, other.right().fold());
-
-                        if (compare != null) {
-                            // AND
-                            if ((conjunctive &&
-                            // a > 3 AND a > 2 -> a > 3
-                                (compare > 0 ||
-                                // a > 2 AND a >= 2 -> a > 2
-                                    (compare == 0 && main instanceof GreaterThan && other instanceof GreaterThanOrEqual))) ||
-                            // OR
-                                (conjunctive == false &&
-                                // a > 2 OR a > 3 -> a > 2
-                                    (compare < 0 ||
-                                    // a >= 2 OR a > 2 -> a >= 2
-                                        (compare == 0 && main instanceof GreaterThanOrEqual && other instanceof GreaterThan)))) {
-                                bcs.remove(i);
-                                bcs.add(i, main);
-                            }
-                            // found a match
-                            return true;
-                        }
-
-                        return false;
-                    }
-                }
-                // if bc is a lower/higher value or lte vs lt, use it instead
-                else if ((other instanceof LessThan || other instanceof LessThanOrEqual)
-                    && (main instanceof LessThan || main instanceof LessThanOrEqual)) {
-
-                        if (main.left().semanticEquals(other.left())) {
-                            Integer compare = BinaryComparison.compare(value, other.right().fold());
-
-                            if (compare != null) {
-                                // AND
-                                if ((conjunctive &&
-                                // a < 2 AND a < 3 -> a < 2
-                                    (compare < 0 ||
-                                // a < 2 AND a <= 2 -> a < 2
-                                        (compare == 0 && main instanceof LessThan && other instanceof LessThanOrEqual))) ||
-                                // OR
-                                    (conjunctive == false &&
-                                // a < 2 OR a < 3 -> a < 3
-                                        (compare > 0 ||
-                                // a <= 2 OR a < 2 -> a <= 2
-                                            (compare == 0 && main instanceof LessThanOrEqual && other instanceof LessThan)))) {
-                                    bcs.remove(i);
-                                    bcs.add(i, main);
-
-                                }
-                                // found a match
-                                return true;
-                            }
-
-                            return false;
-                        }
-                    }
-            }
-
-            return false;
-        }
-
-        private static boolean notEqualsIsRemovableFromConjunction(NotEquals notEquals, List<Range> ranges, List<BinaryComparison> bcs) {
-            Object neqVal = notEquals.right().fold();
-            Integer comp;
-
-            // check on "condition-overlapping" ranges:
-            // a != 2 AND 3 < a < 5 -> 3 < a < 5; a != 2 AND 0 < a < 1 -> 0 < a < 1 (discard NotEquals)
-            // a != 2 AND 2 <= a < 3 -> 2 < a < 3; a != 3 AND 2 < a <= 3 -> 2 < a < 3 (discard NotEquals, plus update Range)
-            // a != 2 AND 1 < a < 3 -> nop (do nothing)
-            for (int i = 0; i < ranges.size(); i++) {
-                Range range = ranges.get(i);
-
-                if (notEquals.left().semanticEquals(range.value())) {
-                    comp = range.lower().foldable() ? BinaryComparison.compare(neqVal, range.lower().fold()) : null;
-                    if (comp != null) {
-                        if (comp <= 0) {
-                            if (comp == 0 && range.includeLower()) { // a != 2 AND 2 <= a < ? -> 2 < a < ?
-                                ranges.set(
-                                    i,
-                                    new Range(
-                                        range.source(),
-                                        range.value(),
-                                        range.lower(),
-                                        false,
-                                        range.upper(),
-                                        range.includeUpper(),
-                                        range.zoneId()
-                                    )
-                                );
-                            }
-                            // else: !.includeLower() : a != 2 AND 2 < a < 3 -> 2 < a < 3; or:
-                            // else: comp < 0 : a != 2 AND 3 < a < ? -> 3 < a < ?
-
-                            return true;
-                        } else { // comp > 0 : a != 4 AND 2 < a < ? : can only remove NotEquals if outside the range
-                            comp = range.upper().foldable() ? BinaryComparison.compare(neqVal, range.upper().fold()) : null;
-                            if (comp != null && comp >= 0) {
-                                if (comp == 0 && range.includeUpper()) { // a != 4 AND 2 < a <= 4 -> 2 < a < 4
-                                    ranges.set(
-                                        i,
-                                        new Range(
-                                            range.source(),
-                                            range.value(),
-                                            range.lower(),
-                                            range.includeLower(),
-                                            range.upper(),
-                                            false,
-                                            range.zoneId()
-                                        )
-                                    );
-                                }
-                                // else: !.includeUpper() : a != 4 AND 2 < a < 4 -> 2 < a < 4
-                                // else: comp > 0 : a != 4 AND 2 < a < 3 -> 2 < a < 3
-
-                                return true;
-                            }
-                            // else: comp < 0 : a != 4 AND 2 < a < 5 -> nop; or:
-                            // else: comp == null : upper bound not comparable -> nop
-                        }
-                    } // else: comp == null : lower bound not comparable: evaluate upper bound, in case non-equality value is ">="
-
-                    comp = range.upper().foldable() ? BinaryComparison.compare(neqVal, range.upper().fold()) : null;
-                    if (comp != null && comp >= 0) {
-                        if (comp == 0 && range.includeUpper()) { // a != 3 AND ?? < a <= 3 -> ?? < a < 3
-                            ranges.set(
-                                i,
-                                new Range(
-                                    range.source(),
-                                    range.value(),
-                                    range.lower(),
-                                    range.includeLower(),
-                                    range.upper(),
-                                    false,
-                                    range.zoneId()
-                                )
-                            );
-                        }
-                        // else: !.includeUpper() : a != 3 AND ?? < a < 3 -> ?? < a < 3
-                        // else: comp > 0 : a != 3 and ?? < a < 2 -> ?? < a < 2
-
-                        return true;
-                    }
-                    // else: comp < 0 : a != 3 AND ?? < a < 4 -> nop, as a decision can't be drawn; or:
-                    // else: comp == null : a != 3 AND ?? < a < ?? -> nop
-                }
-            }
-
-            // check on "condition-overlapping" inequalities:
-            // a != 2 AND a > 3 -> a > 3 (discard NotEquals)
-            // a != 2 AND a >= 2 -> a > 2 (discard NotEquals plus update inequality)
-            // a != 2 AND a > 1 -> nop (do nothing)
-            //
-            // a != 2 AND a < 3 -> nop
-            // a != 2 AND a <= 2 -> a < 2
-            // a != 2 AND a < 1 -> a < 1
-            for (int i = 0; i < bcs.size(); i++) {
-                BinaryComparison bc = bcs.get(i);
-
-                if (notEquals.left().semanticEquals(bc.left())) {
-                    if (bc instanceof LessThan || bc instanceof LessThanOrEqual) {
-                        comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold()) : null;
-                        if (comp != null) {
-                            if (comp >= 0) {
-                                if (comp == 0 && bc instanceof LessThanOrEqual) { // a != 2 AND a <= 2 -> a < 2
-                                    bcs.set(i, new LessThan(bc.source(), bc.left(), bc.right(), bc.zoneId()));
-                                } // else : comp > 0 (a != 2 AND a </<= 1 -> a </<= 1), or == 0 && bc i.of "<" (a != 2 AND a < 2 -> a < 2)
-                                return true;
-                            } // else: comp < 0 : a != 2 AND a </<= 3 -> nop
-                        } // else: non-comparable, nop
-                    } else if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) {
-                        comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold()) : null;
-                        if (comp != null) {
-                            if (comp <= 0) {
-                                if (comp == 0 && bc instanceof GreaterThanOrEqual) { // a != 2 AND a >= 2 -> a > 2
-                                    bcs.set(i, new GreaterThan(bc.source(), bc.left(), bc.right(), bc.zoneId()));
-                                } // else: comp < 0 (a != 2 AND a >/>= 3 -> a >/>= 3), or == 0 && bc i.of ">" (a != 2 AND a > 2 -> a > 2)
-                                return true;
-                            } // else: comp > 0 : a != 2 AND a >/>= 1 -> nop
-                        } // else: non-comparable, nop
-                    } // else: other non-relevant type
-                }
-            }
-
-            return false;
-        }
-
-    }
-
     /**
      * Combine disjunctions on the same field into an In expression.
      * This rule looks for both simple equalities:
@@ -1376,33 +445,6 @@ protected LogicalPlan rule(Limit limit) {
         protected abstract LogicalPlan skipPlan(Limit limit);
     }
 
-    public static class ReplaceRegexMatch extends OptimizerExpressionRule<RegexMatch<?>> {
-
-        public ReplaceRegexMatch() {
-            super(TransformDirection.DOWN);
-        }
-
-        @Override
-        protected Expression rule(RegexMatch<?> regexMatch) {
-            Expression e = regexMatch;
-            StringPattern pattern = regexMatch.pattern();
-            if (pattern.matchesAll()) {
-                e = new IsNotNull(e.source(), regexMatch.field());
-            } else {
-                String match = pattern.exactMatch();
-                if (match != null) {
-                    Literal literal = new Literal(regexMatch.source(), match, DataType.KEYWORD);
-                    e = regexToEquals(regexMatch, literal);
-                }
-            }
-            return e;
-        }
-
-        protected Expression regexToEquals(RegexMatch<?> regexMatch, Literal literal) {
-            return new Equals(regexMatch.source(), regexMatch.field(), literal);
-        }
-    }
-
     public static class FoldNull extends OptimizerExpressionRule<Expression> {
 
         public FoldNull() {
@@ -1410,7 +452,7 @@ public FoldNull() {
         }
 
         @Override
-        protected Expression rule(Expression e) {
+        public Expression rule(Expression e) {
             Expression result = tryReplaceIsNullIsNotNull(e);
             if (result != e) {
                 return result;
@@ -1450,7 +492,7 @@ public PropagateNullable() {
         }
 
         @Override
-        protected Expression rule(And and) {
+        public Expression rule(And and) {
             List<Expression> splits = Predicates.splitAnd(and);
 
             Set<Expression> nullExpressions = new LinkedHashSet<>();
@@ -1529,93 +571,6 @@ protected Expression nonNullify(Expression exp, Expression nonNullExp) {
         }
     }
 
-    /**
-     * Simplify IsNotNull targets by resolving the underlying expression to its root fields with unknown
-     * nullability.
-     * e.g.
-     * (x + 1) / 2 IS NOT NULL --> x IS NOT NULL AND (x+1) / 2 IS NOT NULL
-     * SUBSTRING(x, 3) > 4 IS NOT NULL --> x IS NOT NULL AND SUBSTRING(x, 3) > 4 IS NOT NULL
-     * When dealing with multiple fields, a conjunction/disjunction based on the predicate:
-     * (x + y) / 4 IS NOT NULL --> x IS NOT NULL AND y IS NOT NULL AND (x + y) / 4 IS NOT NULL
-     * This handles the case of fields nested inside functions or expressions in order to avoid:
-     * - having to evaluate the whole expression
-     * - not pushing down the filter due to expression evaluation
-     * IS NULL cannot be simplified since it leads to a disjunction which prevents the filter to be
-     * pushed down:
-     * (x + 1) IS NULL --> x IS NULL OR x + 1 IS NULL
-     * and x IS NULL cannot be pushed down
-     * <br/>
-     * Implementation-wise this rule goes bottom-up, keeping an alias up to date to the current plan
-     * and then looks for replacing the target.
-     */
-    public static class InferIsNotNull extends Rule<LogicalPlan, LogicalPlan> {
-
-        @Override
-        public LogicalPlan apply(LogicalPlan plan) {
-            // the alias map is shared across the whole plan
-            AttributeMap<Expression> aliases = new AttributeMap<>();
-            // traverse bottom-up to pick up the aliases as we go
-            plan = plan.transformUp(p -> inspectPlan(p, aliases));
-            return plan;
-        }
-
-        private LogicalPlan inspectPlan(LogicalPlan plan, AttributeMap<Expression> aliases) {
-            // inspect just this plan properties
-            plan.forEachExpression(Alias.class, a -> aliases.put(a.toAttribute(), a.child()));
-            // now go about finding isNull/isNotNull
-            LogicalPlan newPlan = plan.transformExpressionsOnlyUp(IsNotNull.class, inn -> inferNotNullable(inn, aliases));
-            return newPlan;
-        }
-
-        private Expression inferNotNullable(IsNotNull inn, AttributeMap<Expression> aliases) {
-            Expression result = inn;
-            Set<Expression> refs = resolveExpressionAsRootAttributes(inn.field(), aliases);
-            // no refs found or could not detect - return the original function
-            if (refs.size() > 0) {
-                // add IsNull for the filters along with the initial inn
-                var innList = CollectionUtils.combine(refs.stream().map(r -> (Expression) new IsNotNull(inn.source(), r)).toList(), inn);
-                result = Predicates.combineAnd(innList);
-            }
-            return result;
-        }
-
-        /**
-         * Unroll the expression to its references to get to the root fields
-         * that really matter for filtering.
-         */
-        protected Set<Expression> resolveExpressionAsRootAttributes(Expression exp, AttributeMap<Expression> aliases) {
-            Set<Expression> resolvedExpressions = new LinkedHashSet<>();
-            boolean changed = doResolve(exp, aliases, resolvedExpressions);
-            return changed ? resolvedExpressions : emptySet();
-        }
-
-        private boolean doResolve(Expression exp, AttributeMap<Expression> aliases, Set<Expression> resolvedExpressions) {
-            boolean changed = false;
-            // check if the expression can be skipped or is not nullabe
-            if (skipExpression(exp)) {
-                resolvedExpressions.add(exp);
-            } else {
-                for (Expression e : exp.references()) {
-                    Expression resolved = aliases.resolve(e, e);
-                    // found a root attribute, bail out
-                    if (resolved instanceof Attribute a && resolved == e) {
-                        resolvedExpressions.add(a);
-                        // don't mark things as change if the original expression hasn't been broken down
-                        changed |= resolved != exp;
-                    } else {
-                        // go further
-                        changed |= doResolve(resolved, aliases, resolvedExpressions);
-                    }
-                }
-            }
-            return changed;
-        }
-
-        protected boolean skipExpression(Expression e) {
-            return e.nullable() == Nullability.FALSE;
-        }
-    }
-
     public static final class SetAsOptimized extends Rule<LogicalPlan, LogicalPlan> {
 
         @Override
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
index 84586ed851824..12dbb23a86c59 100644
--- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
+++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java
@@ -8,94 +8,24 @@
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.TestUtils;
-import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
-import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
-import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
-import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
-import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
-import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
-import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
-import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.Add;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.Div;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.Mod;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.Mul;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.Sub;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Equals;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.GreaterThan;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.GreaterThanOrEqual;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.In;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.LessThan;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.LessThanOrEqual;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.NotEquals;
-import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.NullEquals;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.LikePattern;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike;
-import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
-import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BinaryComparisonSimplification;
-import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BooleanFunctionEqualsElimination;
-import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BooleanSimplification;
-import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.CombineBinaryComparisons;
-import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.ConstantFolding;
-import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.FoldNull;
-import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.LiteralsOnTheRight;
-import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.PropagateEquals;
-import org.elasticsearch.xpack.esql.core.plan.logical.EsRelation;
-import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
-import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.util.StringUtils;
 
-import java.time.ZoneId;
 import java.util.Collections;
 import java.util.List;
 
-import static java.util.Arrays.asList;
-import static java.util.Collections.singletonList;
-import static org.elasticsearch.xpack.esql.core.TestUtils.equalsOf;
-import static org.elasticsearch.xpack.esql.core.TestUtils.fieldAttribute;
-import static org.elasticsearch.xpack.esql.core.TestUtils.greaterThanOf;
-import static org.elasticsearch.xpack.esql.core.TestUtils.greaterThanOrEqualOf;
-import static org.elasticsearch.xpack.esql.core.TestUtils.lessThanOf;
-import static org.elasticsearch.xpack.esql.core.TestUtils.lessThanOrEqualOf;
-import static org.elasticsearch.xpack.esql.core.TestUtils.notEqualsOf;
-import static org.elasticsearch.xpack.esql.core.TestUtils.nullEqualsOf;
 import static org.elasticsearch.xpack.esql.core.TestUtils.of;
 import static org.elasticsearch.xpack.esql.core.TestUtils.rangeOf;
-import static org.elasticsearch.xpack.esql.core.TestUtils.relation;
-import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
-import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL;
-import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
-import static org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.CombineDisjunctionsToIn;
-import static org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.PropagateNullable;
-import static org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.ReplaceRegexMatch;
-import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
-import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
-import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
-import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
-import static org.hamcrest.Matchers.contains;
 
 public class OptimizerRulesTests extends ESTestCase {
 
-    private static final Expression DUMMY_EXPRESSION = new DummyBooleanExpression(EMPTY, 0);
-
-    private static final Literal ONE = L(1);
-    private static final Literal TWO = L(2);
-    private static final Literal THREE = L(3);
-    private static final Literal FOUR = L(4);
     private static final Literal FIVE = L(5);
     private static final Literal SIX = L(6);
 
@@ -152,198 +82,6 @@ private static FieldAttribute getFieldAttribute() {
         return TestUtils.getFieldAttribute("a");
     }
 
-    //
-    // Constant folding
-    //
-
-    public void testConstantFolding() {
-        Expression exp = new Add(EMPTY, TWO, THREE);
-
-        assertTrue(exp.foldable());
-        Expression result = new ConstantFolding().rule(exp);
-        assertTrue(result instanceof Literal);
-        assertEquals(5, ((Literal) result).value());
-
-        // check now with an alias
-        result = new ConstantFolding().rule(new Alias(EMPTY, "a", exp));
-        assertEquals("a", Expressions.name(result));
-        assertEquals(Alias.class, result.getClass());
-    }
-
-    public void testConstantFoldingBinaryComparison() {
-        assertEquals(FALSE, new ConstantFolding().rule(greaterThanOf(TWO, THREE)).canonical());
-        assertEquals(FALSE, new ConstantFolding().rule(greaterThanOrEqualOf(TWO, THREE)).canonical());
-        assertEquals(FALSE, new ConstantFolding().rule(equalsOf(TWO, THREE)).canonical());
-        assertEquals(FALSE, new ConstantFolding().rule(nullEqualsOf(TWO, THREE)).canonical());
-        assertEquals(FALSE, new ConstantFolding().rule(nullEqualsOf(TWO, NULL)).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(notEqualsOf(TWO, THREE)).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(lessThanOrEqualOf(TWO, THREE)).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(lessThanOf(TWO, THREE)).canonical());
-    }
-
-    public void testConstantFoldingBinaryLogic() {
-        assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, greaterThanOf(TWO, THREE), TRUE)).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, greaterThanOrEqualOf(TWO, THREE), TRUE)).canonical());
-    }
-
-    public void testConstantFoldingBinaryLogic_WithNullHandling() {
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, NULL, TRUE)).canonical().nullable());
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, TRUE, NULL)).canonical().nullable());
-        assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, NULL, FALSE)).canonical());
-        assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, FALSE, NULL)).canonical());
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, NULL, NULL)).canonical().nullable());
-
-        assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, TRUE)).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, TRUE, NULL)).canonical());
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, FALSE)).canonical().nullable());
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, FALSE, NULL)).canonical().nullable());
-        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, NULL)).canonical().nullable());
-    }
-
-    public void testConstantFoldingRange() {
-        assertEquals(true, new ConstantFolding().rule(rangeOf(FIVE, FIVE, true, L(10), false)).fold());
-        assertEquals(false, new ConstantFolding().rule(rangeOf(FIVE, FIVE, false, L(10), false)).fold());
-    }
-
-    public void testConstantNot() {
-        assertEquals(FALSE, new ConstantFolding().rule(new Not(EMPTY, TRUE)));
-        assertEquals(TRUE, new ConstantFolding().rule(new Not(EMPTY, FALSE)));
-    }
-
-    public void testConstantFoldingLikes() {
-        assertEquals(TRUE, new ConstantFolding().rule(new Like(EMPTY, of("test_emp"), new LikePattern("test%", (char) 0))).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(new WildcardLike(EMPTY, of("test_emp"), new WildcardPattern("test*"))).canonical());
-        assertEquals(TRUE, new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical());
-    }
-
-    public void testArithmeticFolding() {
-        assertEquals(10, foldOperator(new Add(EMPTY, L(7), THREE)));
-        assertEquals(4, foldOperator(new Sub(EMPTY, L(7), THREE)));
-        assertEquals(21, foldOperator(new Mul(EMPTY, L(7), THREE)));
-        assertEquals(2, foldOperator(new Div(EMPTY, L(7), THREE)));
-        assertEquals(1, foldOperator(new Mod(EMPTY, L(7), THREE)));
-    }
-
-    private static Object foldOperator(BinaryOperator<?, ?, ?, ?> b) {
-        return ((Literal) new ConstantFolding().rule(b)).value();
-    }
-
-    //
-    // Logical simplifications
-    //
-
-    public void testLiteralsOnTheRight() {
-        Alias a = new Alias(EMPTY, "a", L(10));
-        Expression result = new LiteralsOnTheRight().rule(equalsOf(FIVE, a));
-        assertTrue(result instanceof Equals);
-        Equals eq = (Equals) result;
-        assertEquals(a, eq.left());
-        assertEquals(FIVE, eq.right());
-
-        a = new Alias(EMPTY, "a", L(10));
-        result = new LiteralsOnTheRight().rule(nullEqualsOf(FIVE, a));
-        assertTrue(result instanceof NullEquals);
-        NullEquals nullEquals = (NullEquals) result;
-        assertEquals(a, nullEquals.left());
-        assertEquals(FIVE, nullEquals.right());
-    }
-
-    public void testBoolSimplifyOr() {
-        BooleanSimplification simplification = new BooleanSimplification();
-
-        assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, TRUE)));
-        assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, DUMMY_EXPRESSION)));
-        assertEquals(TRUE, simplification.rule(new Or(EMPTY, DUMMY_EXPRESSION, TRUE)));
-
-        assertEquals(FALSE, simplification.rule(new Or(EMPTY, FALSE, FALSE)));
-        assertEquals(DUMMY_EXPRESSION, simplification.rule(new Or(EMPTY, FALSE, DUMMY_EXPRESSION)));
-        assertEquals(DUMMY_EXPRESSION, simplification.rule(new Or(EMPTY, DUMMY_EXPRESSION, FALSE)));
-    }
-
-    public void testBoolSimplifyAnd() {
-        BooleanSimplification simplification = new BooleanSimplification();
-
-        assertEquals(TRUE, simplification.rule(new And(EMPTY, TRUE, TRUE)));
-        assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, TRUE, DUMMY_EXPRESSION)));
-        assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, DUMMY_EXPRESSION, TRUE)));
-
-        assertEquals(FALSE, simplification.rule(new And(EMPTY, FALSE, FALSE)));
-        assertEquals(FALSE, simplification.rule(new And(EMPTY, FALSE, DUMMY_EXPRESSION)));
-        assertEquals(FALSE, simplification.rule(new And(EMPTY, DUMMY_EXPRESSION, FALSE)));
-    }
-
-    public void testBoolCommonFactorExtraction() {
-        BooleanSimplification simplification = new BooleanSimplification();
-
-        Expression a1 = new DummyBooleanExpression(EMPTY, 1);
-        Expression a2 = new DummyBooleanExpression(EMPTY, 1);
-        Expression b = new DummyBooleanExpression(EMPTY, 2);
-        Expression c = new DummyBooleanExpression(EMPTY, 3);
-
-        Or actual = new Or(EMPTY, new And(EMPTY, a1, b), new And(EMPTY, a2, c));
-        And expected = new And(EMPTY, a1, new Or(EMPTY, b, c));
-
-        assertEquals(expected, simplification.rule(actual));
-    }
-
-    public void testBinaryComparisonSimplification() {
-        assertEquals(TRUE, new BinaryComparisonSimplification().rule(equalsOf(FIVE, FIVE)));
-        assertEquals(TRUE, new BinaryComparisonSimplification().rule(nullEqualsOf(FIVE, FIVE)));
-        assertEquals(TRUE, new BinaryComparisonSimplification().rule(nullEqualsOf(NULL, NULL)));
-        assertEquals(FALSE, new BinaryComparisonSimplification().rule(notEqualsOf(FIVE, FIVE)));
-        assertEquals(TRUE, new BinaryComparisonSimplification().rule(greaterThanOrEqualOf(FIVE, FIVE)));
-        assertEquals(TRUE, new BinaryComparisonSimplification().rule(lessThanOrEqualOf(FIVE, FIVE)));
-
-        assertEquals(FALSE, new BinaryComparisonSimplification().rule(greaterThanOf(FIVE, FIVE)));
-        assertEquals(FALSE, new BinaryComparisonSimplification().rule(lessThanOf(FIVE, FIVE)));
-    }
-
-    public void testNullEqualsWithNullLiteralBecomesIsNull() {
-        LiteralsOnTheRight swapLiteralsToRight = new LiteralsOnTheRight();
-        BinaryComparisonSimplification bcSimpl = new BinaryComparisonSimplification();
-        FieldAttribute fa = getFieldAttribute();
-        Source source = new Source(1, 10, "IS_NULL(a)");
-
-        Expression e = bcSimpl.rule((BinaryComparison) swapLiteralsToRight.rule(new NullEquals(source, fa, NULL, randomZone())));
-        assertEquals(IsNull.class, e.getClass());
-        IsNull isNull = (IsNull) e;
-        assertEquals(source, isNull.source());
-
-        e = bcSimpl.rule((BinaryComparison) swapLiteralsToRight.rule(new NullEquals(source, NULL, fa, randomZone())));
-        assertEquals(IsNull.class, e.getClass());
-        isNull = (IsNull) e;
-        assertEquals(source, isNull.source());
-    }
-
-    public void testBoolEqualsSimplificationOnExpressions() {
-        BooleanFunctionEqualsElimination s = new BooleanFunctionEqualsElimination();
-        Expression exp = new GreaterThan(EMPTY, getFieldAttribute(), L(0), null);
-
-        assertEquals(exp, s.rule(new Equals(EMPTY, exp, TRUE)));
-        assertEquals(new Not(EMPTY, exp), s.rule(new Equals(EMPTY, exp, FALSE)));
-    }
-
-    public void testBoolEqualsSimplificationOnFields() {
-        BooleanFunctionEqualsElimination s = new BooleanFunctionEqualsElimination();
-
-        FieldAttribute field = getFieldAttribute();
-
-        List<? extends BinaryComparison> comparisons = asList(
-            new Equals(EMPTY, field, TRUE),
-            new Equals(EMPTY, field, FALSE),
-            notEqualsOf(field, TRUE),
-            notEqualsOf(field, FALSE),
-            new Equals(EMPTY, NULL, TRUE),
-            new Equals(EMPTY, NULL, FALSE),
-            notEqualsOf(NULL, TRUE),
-            notEqualsOf(NULL, FALSE)
-        );
-
-        for (BinaryComparison comparison : comparisons) {
-            assertEquals(comparison, s.rule(comparison));
-        }
-    }
-
     //
     // Range optimization
     //
@@ -368,1399 +106,4 @@ public void testFoldExcludingRangeWithDifferentTypesToFalse() {
 
     // Conjunction
 
-    public void testCombineBinaryComparisonsNotComparable() {
-        FieldAttribute fa = getFieldAttribute();
-        LessThanOrEqual lte = lessThanOrEqualOf(fa, SIX);
-        LessThan lt = lessThanOf(fa, FALSE);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        And and = new And(EMPTY, lte, lt);
-        Expression exp = rule.rule(and);
-        assertEquals(exp, and);
-    }
-
-    // a <= 6 AND a < 5 -> a < 5
-    public void testCombineBinaryComparisonsUpper() {
-        FieldAttribute fa = getFieldAttribute();
-        LessThanOrEqual lte = lessThanOrEqualOf(fa, SIX);
-        LessThan lt = lessThanOf(fa, FIVE);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
-        Expression exp = rule.rule(new And(EMPTY, lte, lt));
-        assertEquals(LessThan.class, exp.getClass());
-        LessThan r = (LessThan) exp;
-        assertEquals(FIVE, r.right());
-    }
-
-    // 6 <= a AND 5 < a -> 6 <= a
-    public void testCombineBinaryComparisonsLower() {
-        FieldAttribute fa = getFieldAttribute();
-        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, SIX);
-        GreaterThan gt = greaterThanOf(fa, FIVE);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
-        Expression exp = rule.rule(new And(EMPTY, gte, gt));
-        assertEquals(GreaterThanOrEqual.class, exp.getClass());
-        GreaterThanOrEqual r = (GreaterThanOrEqual) exp;
-        assertEquals(SIX, r.right());
-    }
-
-    // 5 <= a AND 5 < a -> 5 < a
-    public void testCombineBinaryComparisonsInclude() {
-        FieldAttribute fa = getFieldAttribute();
-        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, FIVE);
-        GreaterThan gt = greaterThanOf(fa, FIVE);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
-        Expression exp = rule.rule(new And(EMPTY, gte, gt));
-        assertEquals(GreaterThan.class, exp.getClass());
-        GreaterThan r = (GreaterThan) exp;
-        assertEquals(FIVE, r.right());
-    }
-
-    // 2 < a AND (2 <= a < 3) -> 2 < a < 3
-    public void testCombineBinaryComparisonsAndRangeLower() {
-        FieldAttribute fa = getFieldAttribute();
-
-        GreaterThan gt = greaterThanOf(fa, TWO);
-        Range range = rangeOf(fa, TWO, true, THREE, false);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(new And(EMPTY, gt, range));
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(TWO, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(THREE, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // a < 4 AND (1 < a < 3) -> 1 < a < 3
-    public void testCombineBinaryComparisonsAndRangeUpper() {
-        FieldAttribute fa = getFieldAttribute();
-
-        LessThan lt = lessThanOf(fa, FOUR);
-        Range range = rangeOf(fa, ONE, false, THREE, false);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(new And(EMPTY, range, lt));
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(ONE, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(THREE, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // a <= 2 AND (1 < a < 3) -> 1 < a <= 2
-    public void testCombineBinaryComparisonsAndRangeUpperEqual() {
-        FieldAttribute fa = getFieldAttribute();
-
-        LessThanOrEqual lte = lessThanOrEqualOf(fa, TWO);
-        Range range = rangeOf(fa, ONE, false, THREE, false);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(new And(EMPTY, lte, range));
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(ONE, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(TWO, r.upper());
-        assertTrue(r.includeUpper());
-    }
-
-    // 3 <= a AND 4 < a AND a <= 7 AND a < 6 -> 4 < a < 6
-    public void testCombineMultipleBinaryComparisons() {
-        FieldAttribute fa = getFieldAttribute();
-        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, THREE);
-        GreaterThan gt = greaterThanOf(fa, FOUR);
-        LessThanOrEqual lte = lessThanOrEqualOf(fa, L(7));
-        LessThan lt = lessThanOf(fa, SIX);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
-        Expression exp = rule.rule(new And(EMPTY, gte, new And(EMPTY, gt, new And(EMPTY, lt, lte))));
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(FOUR, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(SIX, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // 3 <= a AND TRUE AND 4 < a AND a != 5 AND a <= 7 -> 4 < a <= 7 AND a != 5 AND TRUE
-    public void testCombineMixedMultipleBinaryComparisons() {
-        FieldAttribute fa = getFieldAttribute();
-        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, THREE);
-        GreaterThan gt = greaterThanOf(fa, FOUR);
-        LessThanOrEqual lte = lessThanOrEqualOf(fa, L(7));
-        Expression ne = new Not(EMPTY, equalsOf(fa, FIVE));
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-
-        // TRUE AND a != 5 AND 4 < a <= 7
-        Expression exp = rule.rule(new And(EMPTY, gte, new And(EMPTY, TRUE, new And(EMPTY, gt, new And(EMPTY, ne, lte)))));
-        assertEquals(And.class, exp.getClass());
-        And and = ((And) exp);
-        assertEquals(Range.class, and.right().getClass());
-        Range r = (Range) and.right();
-        assertEquals(FOUR, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(L(7), r.upper());
-        assertTrue(r.includeUpper());
-    }
-
-    // 1 <= a AND a < 5 -> 1 <= a < 5
-    public void testCombineComparisonsIntoRange() {
-        FieldAttribute fa = getFieldAttribute();
-        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, ONE);
-        LessThan lt = lessThanOf(fa, FIVE);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(new And(EMPTY, gte, lt));
-        assertEquals(Range.class, exp.getClass());
-
-        Range r = (Range) exp;
-        assertEquals(ONE, r.lower());
-        assertTrue(r.includeLower());
-        assertEquals(FIVE, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // 1 < a AND a < 3 AND 2 < b AND b < 4 AND c < 4 -> (1 < a < 3) AND (2 < b < 4) AND c < 4
-    public void testCombineMultipleComparisonsIntoRange() {
-        FieldAttribute fa = TestUtils.getFieldAttribute("a");
-        FieldAttribute fb = TestUtils.getFieldAttribute("b");
-        FieldAttribute fc = TestUtils.getFieldAttribute("c");
-
-        ZoneId zoneId = randomZone();
-        GreaterThan agt1 = new GreaterThan(EMPTY, fa, ONE, zoneId);
-        LessThan alt3 = new LessThan(EMPTY, fa, THREE, zoneId);
-        GreaterThan bgt2 = new GreaterThan(EMPTY, fb, TWO, zoneId);
-        LessThan blt4 = new LessThan(EMPTY, fb, FOUR, zoneId);
-        LessThan clt4 = new LessThan(EMPTY, fc, FOUR, zoneId);
-
-        Expression inputAnd = Predicates.combineAnd(asList(agt1, alt3, bgt2, blt4, clt4));
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression outputAnd = rule.rule((And) inputAnd);
-
-        Range agt1lt3 = new Range(EMPTY, fa, ONE, false, THREE, false, zoneId);
-        Range bgt2lt4 = new Range(EMPTY, fb, TWO, false, FOUR, false, zoneId);
-
-        // The actual outcome is (c < 4) AND (1 < a < 3) AND (2 < b < 4), due to the way the Expression types are combined in the Optimizer
-        Expression expectedAnd = Predicates.combineAnd(asList(clt4, agt1lt3, bgt2lt4));
-
-        assertTrue(outputAnd.semanticEquals(expectedAnd));
-    }
-
-    // (2 < a < 3) AND (1 < a < 4) -> (2 < a < 3)
-    public void testCombineBinaryComparisonsConjunctionOfIncludedRange() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r1 = rangeOf(fa, TWO, false, THREE, false);
-        Range r2 = rangeOf(fa, ONE, false, FOUR, false);
-
-        And and = new And(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(r1, exp);
-    }
-
-    // (2 < a < 3) AND a < 2 -> 2 < a < 2
-    public void testCombineBinaryComparisonsConjunctionOfNonOverlappingBoundaries() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r1 = rangeOf(fa, TWO, false, THREE, false);
-        Range r2 = rangeOf(fa, ONE, false, TWO, false);
-
-        And and = new And(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(TWO, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(TWO, r.upper());
-        assertFalse(r.includeUpper());
-        assertEquals(Boolean.FALSE, r.fold());
-    }
-
-    // (2 < a < 3) AND (2 < a <= 3) -> 2 < a < 3
-    public void testCombineBinaryComparisonsConjunctionOfUpperEqualsOverlappingBoundaries() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r1 = rangeOf(fa, TWO, false, THREE, false);
-        Range r2 = rangeOf(fa, TWO, false, THREE, true);
-
-        And and = new And(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(r1, exp);
-    }
-
-    // (2 < a < 3) AND (1 < a < 3) -> 2 < a < 3
-    public void testCombineBinaryComparisonsConjunctionOverlappingUpperBoundary() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r2 = rangeOf(fa, TWO, false, THREE, false);
-        Range r1 = rangeOf(fa, ONE, false, THREE, false);
-
-        And and = new And(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(r2, exp);
-    }
-
-    // (2 < a <= 3) AND (1 < a < 3) -> 2 < a < 3
-    public void testCombineBinaryComparisonsConjunctionWithDifferentUpperLimitInclusion() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r1 = rangeOf(fa, ONE, false, THREE, false);
-        Range r2 = rangeOf(fa, TWO, false, THREE, true);
-
-        And and = new And(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(TWO, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(THREE, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // (0 < a <= 1) AND (0 <= a < 2) -> 0 < a <= 1
-    public void testRangesOverlappingConjunctionNoLowerBoundary() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r1 = rangeOf(fa, L(0), false, ONE, true);
-        Range r2 = rangeOf(fa, L(0), true, TWO, false);
-
-        And and = new And(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(r1, exp);
-    }
-
-    // a != 2 AND 3 < a < 5 -> 3 < a < 5
-    public void testCombineBinaryComparisonsConjunction_Neq2AndRangeGt3Lt5() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, TWO);
-        Range range = rangeOf(fa, THREE, false, FIVE, false);
-        And and = new And(EMPTY, range, neq);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(THREE, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(FIVE, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // a != 2 AND 0 < a < 1 -> 0 < a < 1
-    public void testCombineBinaryComparisonsConjunction_Neq2AndRangeGt0Lt1() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, TWO);
-        Range range = rangeOf(fa, L(0), false, ONE, false);
-        And and = new And(EMPTY, neq, range);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(L(0), r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(ONE, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // a != 2 AND 2 <= a < 3 -> 2 < a < 3
-    public void testCombineBinaryComparisonsConjunction_Neq2AndRangeGte2Lt3() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, TWO);
-        Range range = rangeOf(fa, TWO, true, THREE, false);
-        And and = new And(EMPTY, neq, range);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(TWO, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(THREE, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // a != 3 AND 2 < a <= 3 -> 2 < a < 3
-    public void testCombineBinaryComparisonsConjunction_Neq3AndRangeGt2Lte3() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, THREE);
-        Range range = rangeOf(fa, TWO, false, THREE, true);
-        And and = new And(EMPTY, neq, range);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(TWO, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(THREE, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // a != 2 AND 1 < a < 3
-    public void testCombineBinaryComparisonsConjunction_Neq2AndRangeGt1Lt3() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, TWO);
-        Range range = rangeOf(fa, ONE, false, THREE, false);
-        And and = new And(EMPTY, neq, range);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(And.class, exp.getClass()); // can't optimize
-    }
-
-    // a != 2 AND a > 3 -> a > 3
-    public void testCombineBinaryComparisonsConjunction_Neq2AndGt3() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, TWO);
-        GreaterThan gt = greaterThanOf(fa, THREE);
-        And and = new And(EMPTY, neq, gt);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(gt, exp);
-    }
-
-    // a != 2 AND a >= 2 -> a > 2
-    public void testCombineBinaryComparisonsConjunction_Neq2AndGte2() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, TWO);
-        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, TWO);
-        And and = new And(EMPTY, neq, gte);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(GreaterThan.class, exp.getClass());
-        GreaterThan gt = (GreaterThan) exp;
-        assertEquals(TWO, gt.right());
-    }
-
-    // a != 2 AND a >= 1 -> nop
-    public void testCombineBinaryComparisonsConjunction_Neq2AndGte1() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, TWO);
-        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, ONE);
-        And and = new And(EMPTY, neq, gte);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(And.class, exp.getClass()); // can't optimize
-    }
-
-    // a != 2 AND a <= 3 -> nop
-    public void testCombineBinaryComparisonsConjunction_Neq2AndLte3() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, TWO);
-        LessThanOrEqual lte = lessThanOrEqualOf(fa, THREE);
-        And and = new And(EMPTY, neq, lte);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(and, exp); // can't optimize
-    }
-
-    // a != 2 AND a <= 2 -> a < 2
-    public void testCombineBinaryComparisonsConjunction_Neq2AndLte2() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, TWO);
-        LessThanOrEqual lte = lessThanOrEqualOf(fa, TWO);
-        And and = new And(EMPTY, neq, lte);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(LessThan.class, exp.getClass());
-        LessThan lt = (LessThan) exp;
-        assertEquals(TWO, lt.right());
-    }
-
-    // a != 2 AND a <= 1 -> a <= 1
-    public void testCombineBinaryComparisonsConjunction_Neq2AndLte1() {
-        FieldAttribute fa = getFieldAttribute();
-
-        NotEquals neq = notEqualsOf(fa, TWO);
-        LessThanOrEqual lte = lessThanOrEqualOf(fa, ONE);
-        And and = new And(EMPTY, neq, lte);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(and);
-        assertEquals(lte, exp);
-    }
-
-    // Disjunction
-
-    public void testCombineBinaryComparisonsDisjunctionNotComparable() {
-        FieldAttribute fa = getFieldAttribute();
-
-        GreaterThan gt1 = greaterThanOf(fa, ONE);
-        GreaterThan gt2 = greaterThanOf(fa, FALSE);
-
-        Or or = new Or(EMPTY, gt1, gt2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(exp, or);
-    }
-
-    // 2 < a OR 1 < a OR 3 < a -> 1 < a
-    public void testCombineBinaryComparisonsDisjunctionLowerBound() {
-        FieldAttribute fa = getFieldAttribute();
-
-        GreaterThan gt1 = greaterThanOf(fa, ONE);
-        GreaterThan gt2 = greaterThanOf(fa, TWO);
-        GreaterThan gt3 = greaterThanOf(fa, THREE);
-
-        Or or = new Or(EMPTY, gt1, new Or(EMPTY, gt2, gt3));
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(GreaterThan.class, exp.getClass());
-
-        GreaterThan gt = (GreaterThan) exp;
-        assertEquals(ONE, gt.right());
-    }
-
-    // 2 < a OR 1 < a OR 3 <= a -> 1 < a
-    public void testCombineBinaryComparisonsDisjunctionIncludeLowerBounds() {
-        FieldAttribute fa = getFieldAttribute();
-
-        GreaterThan gt1 = greaterThanOf(fa, ONE);
-        GreaterThan gt2 = greaterThanOf(fa, TWO);
-        GreaterThanOrEqual gte3 = greaterThanOrEqualOf(fa, THREE);
-
-        Or or = new Or(EMPTY, new Or(EMPTY, gt1, gt2), gte3);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(GreaterThan.class, exp.getClass());
-
-        GreaterThan gt = (GreaterThan) exp;
-        assertEquals(ONE, gt.right());
-    }
-
-    // a < 1 OR a < 2 OR a < 3 -> a < 3
-    public void testCombineBinaryComparisonsDisjunctionUpperBound() {
-        FieldAttribute fa = getFieldAttribute();
-
-        LessThan lt1 = lessThanOf(fa, ONE);
-        LessThan lt2 = lessThanOf(fa, TWO);
-        LessThan lt3 = lessThanOf(fa, THREE);
-
-        Or or = new Or(EMPTY, new Or(EMPTY, lt1, lt2), lt3);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(LessThan.class, exp.getClass());
-
-        LessThan lt = (LessThan) exp;
-        assertEquals(THREE, lt.right());
-    }
-
-    // a < 2 OR a <= 2 OR a < 1 -> a <= 2
-    public void testCombineBinaryComparisonsDisjunctionIncludeUpperBounds() {
-        FieldAttribute fa = getFieldAttribute();
-
-        LessThan lt1 = lessThanOf(fa, ONE);
-        LessThan lt2 = lessThanOf(fa, TWO);
-        LessThanOrEqual lte2 = lessThanOrEqualOf(fa, TWO);
-
-        Or or = new Or(EMPTY, lt2, new Or(EMPTY, lte2, lt1));
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(LessThanOrEqual.class, exp.getClass());
-
-        LessThanOrEqual lte = (LessThanOrEqual) exp;
-        assertEquals(TWO, lte.right());
-    }
-
-    // a < 2 OR 3 < a OR a < 1 OR 4 < a -> a < 2 OR 3 < a
-    public void testCombineBinaryComparisonsDisjunctionOfLowerAndUpperBounds() {
-        FieldAttribute fa = getFieldAttribute();
-
-        LessThan lt1 = lessThanOf(fa, ONE);
-        LessThan lt2 = lessThanOf(fa, TWO);
-
-        GreaterThan gt3 = greaterThanOf(fa, THREE);
-        GreaterThan gt4 = greaterThanOf(fa, FOUR);
-
-        Or or = new Or(EMPTY, new Or(EMPTY, lt2, gt3), new Or(EMPTY, lt1, gt4));
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(Or.class, exp.getClass());
-
-        Or ro = (Or) exp;
-
-        assertEquals(LessThan.class, ro.left().getClass());
-        LessThan lt = (LessThan) ro.left();
-        assertEquals(TWO, lt.right());
-        assertEquals(GreaterThan.class, ro.right().getClass());
-        GreaterThan gt = (GreaterThan) ro.right();
-        assertEquals(THREE, gt.right());
-    }
-
-    // (2 < a < 3) OR (1 < a < 4) -> (1 < a < 4)
-    public void testCombineBinaryComparisonsDisjunctionOfIncludedRangeNotComparable() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r1 = rangeOf(fa, TWO, false, THREE, false);
-        Range r2 = rangeOf(fa, ONE, false, FALSE, false);
-
-        Or or = new Or(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(or, exp);
-    }
-
-    // (2 < a < 3) OR (1 < a < 4) -> (1 < a < 4)
-    public void testCombineBinaryComparisonsDisjunctionOfIncludedRange() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r1 = rangeOf(fa, TWO, false, THREE, false);
-        Range r2 = rangeOf(fa, ONE, false, FOUR, false);
-
-        Or or = new Or(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(Range.class, exp.getClass());
-
-        Range r = (Range) exp;
-        assertEquals(ONE, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(FOUR, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // (2 < a < 3) OR (1 < a < 2) -> same
-    public void testCombineBinaryComparisonsDisjunctionOfNonOverlappingBoundaries() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r1 = rangeOf(fa, TWO, false, THREE, false);
-        Range r2 = rangeOf(fa, ONE, false, TWO, false);
-
-        Or or = new Or(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(or, exp);
-    }
-
-    // (2 < a < 3) OR (2 < a <= 3) -> 2 < a <= 3
-    public void testCombineBinaryComparisonsDisjunctionOfUpperEqualsOverlappingBoundaries() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r1 = rangeOf(fa, TWO, false, THREE, false);
-        Range r2 = rangeOf(fa, TWO, false, THREE, true);
-
-        Or or = new Or(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(r2, exp);
-    }
-
-    // (2 < a < 3) OR (1 < a < 3) -> 1 < a < 3
-    public void testCombineBinaryComparisonsOverlappingUpperBoundary() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r2 = rangeOf(fa, TWO, false, THREE, false);
-        Range r1 = rangeOf(fa, ONE, false, THREE, false);
-
-        Or or = new Or(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(r1, exp);
-    }
-
-    // (2 < a <= 3) OR (1 < a < 3) -> same (the <= prevents the ranges from being combined)
-    public void testCombineBinaryComparisonsWithDifferentUpperLimitInclusion() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r1 = rangeOf(fa, ONE, false, THREE, false);
-        Range r2 = rangeOf(fa, TWO, false, THREE, true);
-
-        Or or = new Or(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(or, exp);
-    }
-
-    // (a = 1 AND b = 3 AND c = 4) OR (a = 2 AND b = 3 AND c = 4) -> (b = 3 AND c = 4) AND (a = 1 OR a = 2)
-    public void testBooleanSimplificationCommonExpressionSubstraction() {
-        FieldAttribute fa = TestUtils.getFieldAttribute("a");
-        FieldAttribute fb = TestUtils.getFieldAttribute("b");
-        FieldAttribute fc = TestUtils.getFieldAttribute("c");
-
-        Expression a1 = equalsOf(fa, ONE);
-        Expression a2 = equalsOf(fa, TWO);
-        And common = new And(EMPTY, equalsOf(fb, THREE), equalsOf(fc, FOUR));
-        And left = new And(EMPTY, a1, common);
-        And right = new And(EMPTY, a2, common);
-        Or or = new Or(EMPTY, left, right);
-
-        Expression exp = new BooleanSimplification().rule(or);
-        assertEquals(new And(EMPTY, common, new Or(EMPTY, a1, a2)), exp);
-    }
-
-    // (0 < a <= 1) OR (0 < a < 2) -> 0 < a < 2
-    public void testRangesOverlappingNoLowerBoundary() {
-        FieldAttribute fa = getFieldAttribute();
-
-        Range r2 = rangeOf(fa, L(0), false, TWO, false);
-        Range r1 = rangeOf(fa, L(0), false, ONE, true);
-
-        Or or = new Or(EMPTY, r1, r2);
-
-        CombineBinaryComparisons rule = new CombineBinaryComparisons();
-        Expression exp = rule.rule(or);
-        assertEquals(r2, exp);
-    }
-
-    public void testBinaryComparisonAndOutOfRangeNotEqualsDifferentFields() {
-        FieldAttribute doubleOne = fieldAttribute("double", DOUBLE);
-        FieldAttribute doubleTwo = fieldAttribute("double2", DOUBLE);
-        FieldAttribute intOne = fieldAttribute("int", INTEGER);
-        FieldAttribute datetimeOne = fieldAttribute("datetime", INTEGER);
-        FieldAttribute keywordOne = fieldAttribute("keyword", KEYWORD);
-        FieldAttribute keywordTwo = fieldAttribute("keyword2", KEYWORD);
-
-        List<And> testCases = asList(
-            // double > 10 AND integer != -10
-            new And(EMPTY, greaterThanOf(doubleOne, L(10)), notEqualsOf(intOne, L(-10))),
-            // keyword > '5' AND keyword2 != '48'
-            new And(EMPTY, greaterThanOf(keywordOne, L("5")), notEqualsOf(keywordTwo, L("48"))),
-            // keyword != '2021' AND datetime <= '2020-12-04T17:48:22.954240Z'
-            new And(EMPTY, notEqualsOf(keywordOne, L("2021")), lessThanOrEqualOf(datetimeOne, L("2020-12-04T17:48:22.954240Z"))),
-            // double > 10.1 AND double2 != -10.1
-            new And(EMPTY, greaterThanOf(doubleOne, L(10.1d)), notEqualsOf(doubleTwo, L(-10.1d)))
-        );
-
-        for (And and : testCases) {
-            CombineBinaryComparisons rule = new CombineBinaryComparisons();
-            Expression exp = rule.rule(and);
-            assertEquals("Rule should not have transformed [" + and.nodeString() + "]", and, exp);
-        }
-    }
-
-    // Equals & NullEquals
-
-    // 1 <= a < 10 AND a == 1 -> a == 1
-    public void testEliminateRangeByEqualsInInterval() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq1 = equalsOf(fa, ONE);
-        Range r = rangeOf(fa, ONE, true, L(10), false);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq1, r));
-        assertEquals(eq1, exp);
-    }
-
-    // 1 <= a < 10 AND a <=> 1 -> a <=> 1
-    public void testEliminateRangeByNullEqualsInInterval() {
-        FieldAttribute fa = getFieldAttribute();
-        NullEquals eq1 = nullEqualsOf(fa, ONE);
-        Range r = rangeOf(fa, ONE, true, L(10), false);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq1, r));
-        assertEquals(eq1, exp);
-    }
-
-    // The following tests should work only to simplify filters and
-    // not if the expressions are part of a projection
-    // See: https://github.com/elastic/elasticsearch/issues/35859
-
-    // a == 1 AND a == 2 -> FALSE
-    public void testDualEqualsConjunction() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq1 = equalsOf(fa, ONE);
-        Equals eq2 = equalsOf(fa, TWO);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq1, eq2));
-        assertEquals(FALSE, exp);
-    }
-
-    // a <=> 1 AND a <=> 2 -> FALSE
-    public void testDualNullEqualsConjunction() {
-        FieldAttribute fa = getFieldAttribute();
-        NullEquals eq1 = nullEqualsOf(fa, ONE);
-        NullEquals eq2 = nullEqualsOf(fa, TWO);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq1, eq2));
-        assertEquals(FALSE, exp);
-    }
-
-    // 1 < a < 10 AND a == 10 -> FALSE
-    public void testEliminateRangeByEqualsOutsideInterval() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq1 = equalsOf(fa, L(10));
-        Range r = rangeOf(fa, ONE, false, L(10), false);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq1, r));
-        assertEquals(FALSE, exp);
-    }
-
-    // 1 < a < 10 AND a <=> 10 -> FALSE
-    public void testEliminateRangeByNullEqualsOutsideInterval() {
-        FieldAttribute fa = getFieldAttribute();
-        NullEquals eq1 = nullEqualsOf(fa, L(10));
-        Range r = rangeOf(fa, ONE, false, L(10), false);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq1, r));
-        assertEquals(FALSE, exp);
-    }
-
-    // a != 3 AND a = 3 -> FALSE
-    public void testPropagateEquals_VarNeq3AndVarEq3() {
-        FieldAttribute fa = getFieldAttribute();
-        NotEquals neq = notEqualsOf(fa, THREE);
-        Equals eq = equalsOf(fa, THREE);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, neq, eq));
-        assertEquals(FALSE, exp);
-    }
-
-    // a != 4 AND a = 3 -> a = 3
-    public void testPropagateEquals_VarNeq4AndVarEq3() {
-        FieldAttribute fa = getFieldAttribute();
-        NotEquals neq = notEqualsOf(fa, FOUR);
-        Equals eq = equalsOf(fa, THREE);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, neq, eq));
-        assertEquals(Equals.class, exp.getClass());
-        assertEquals(eq, exp);
-    }
-
-    // a = 2 AND a < 2 -> FALSE
-    public void testPropagateEquals_VarEq2AndVarLt2() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        LessThan lt = lessThanOf(fa, TWO);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, lt));
-        assertEquals(FALSE, exp);
-    }
-
-    // a = 2 AND a <= 2 -> a = 2
-    public void testPropagateEquals_VarEq2AndVarLte2() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        LessThanOrEqual lt = lessThanOrEqualOf(fa, TWO);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, lt));
-        assertEquals(eq, exp);
-    }
-
-    // a = 2 AND a <= 1 -> FALSE
-    public void testPropagateEquals_VarEq2AndVarLte1() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        LessThanOrEqual lt = lessThanOrEqualOf(fa, ONE);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, lt));
-        assertEquals(FALSE, exp);
-    }
-
-    // a = 2 AND a > 2 -> FALSE
-    public void testPropagateEquals_VarEq2AndVarGt2() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        GreaterThan gt = greaterThanOf(fa, TWO);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, gt));
-        assertEquals(FALSE, exp);
-    }
-
-    // a = 2 AND a >= 2 -> a = 2
-    public void testPropagateEquals_VarEq2AndVarGte2() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, TWO);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, gte));
-        assertEquals(eq, exp);
-    }
-
-    // a = 2 AND a > 3 -> FALSE
-    public void testPropagateEquals_VarEq2AndVarLt3() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        GreaterThan gt = greaterThanOf(fa, THREE);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new And(EMPTY, eq, gt));
-        assertEquals(FALSE, exp);
-    }
-
-    // a = 2 AND a < 3 AND a > 1 AND a != 4 -> a = 2
-    public void testPropagateEquals_VarEq2AndVarLt3AndVarGt1AndVarNeq4() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        LessThan lt = lessThanOf(fa, THREE);
-        GreaterThan gt = greaterThanOf(fa, ONE);
-        NotEquals neq = notEqualsOf(fa, FOUR);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression and = Predicates.combineAnd(asList(eq, lt, gt, neq));
-        Expression exp = rule.rule((And) and);
-        assertEquals(eq, exp);
-    }
-
-    // a = 2 AND 1 < a < 3 AND a > 0 AND a != 4 -> a = 2
-    public void testPropagateEquals_VarEq2AndVarRangeGt1Lt3AndVarGt0AndVarNeq4() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        Range range = rangeOf(fa, ONE, false, THREE, false);
-        GreaterThan gt = greaterThanOf(fa, L(0));
-        NotEquals neq = notEqualsOf(fa, FOUR);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression and = Predicates.combineAnd(asList(eq, range, gt, neq));
-        Expression exp = rule.rule((And) and);
-        assertEquals(eq, exp);
-    }
-
-    // a = 2 OR a > 1 -> a > 1
-    public void testPropagateEquals_VarEq2OrVarGt1() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        GreaterThan gt = greaterThanOf(fa, ONE);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, gt));
-        assertEquals(gt, exp);
-    }
-
-    // a = 2 OR a > 2 -> a >= 2
-    public void testPropagateEquals_VarEq2OrVarGte2() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        GreaterThan gt = greaterThanOf(fa, TWO);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, gt));
-        assertEquals(GreaterThanOrEqual.class, exp.getClass());
-        GreaterThanOrEqual gte = (GreaterThanOrEqual) exp;
-        assertEquals(TWO, gte.right());
-    }
-
-    // a = 2 OR a < 3 -> a < 3
-    public void testPropagateEquals_VarEq2OrVarLt3() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        LessThan lt = lessThanOf(fa, THREE);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, lt));
-        assertEquals(lt, exp);
-    }
-
-    // a = 3 OR a < 3 -> a <= 3
-    public void testPropagateEquals_VarEq3OrVarLt3() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, THREE);
-        LessThan lt = lessThanOf(fa, THREE);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, lt));
-        assertEquals(LessThanOrEqual.class, exp.getClass());
-        LessThanOrEqual lte = (LessThanOrEqual) exp;
-        assertEquals(THREE, lte.right());
-    }
-
-    // a = 2 OR 1 < a < 3 -> 1 < a < 3
-    public void testPropagateEquals_VarEq2OrVarRangeGt1Lt3() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        Range range = rangeOf(fa, ONE, false, THREE, false);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, range));
-        assertEquals(range, exp);
-    }
-
-    // a = 2 OR 2 < a < 3 -> 2 <= a < 3
-    public void testPropagateEquals_VarEq2OrVarRangeGt2Lt3() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        Range range = rangeOf(fa, TWO, false, THREE, false);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, range));
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(TWO, r.lower());
-        assertTrue(r.includeLower());
-        assertEquals(THREE, r.upper());
-        assertFalse(r.includeUpper());
-    }
-
-    // a = 3 OR 2 < a < 3 -> 2 < a <= 3
-    public void testPropagateEquals_VarEq3OrVarRangeGt2Lt3() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, THREE);
-        Range range = rangeOf(fa, TWO, false, THREE, false);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, range));
-        assertEquals(Range.class, exp.getClass());
-        Range r = (Range) exp;
-        assertEquals(TWO, r.lower());
-        assertFalse(r.includeLower());
-        assertEquals(THREE, r.upper());
-        assertTrue(r.includeUpper());
-    }
-
-    // a = 2 OR a != 2 -> TRUE
-    public void testPropagateEquals_VarEq2OrVarNeq2() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        NotEquals neq = notEqualsOf(fa, TWO);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, neq));
-        assertEquals(TRUE, exp);
-    }
-
-    // a = 2 OR a != 5 -> a != 5
-    public void testPropagateEquals_VarEq2OrVarNeq5() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        NotEquals neq = notEqualsOf(fa, FIVE);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(new Or(EMPTY, eq, neq));
-        assertEquals(NotEquals.class, exp.getClass());
-        NotEquals ne = (NotEquals) exp;
-        assertEquals(FIVE, ne.right());
-    }
-
-    // a = 2 OR 3 < a < 4 OR a > 2 OR a!= 2 -> TRUE
-    public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() {
-        FieldAttribute fa = getFieldAttribute();
-        Equals eq = equalsOf(fa, TWO);
-        Range range = rangeOf(fa, THREE, false, FOUR, false);
-        GreaterThan gt = greaterThanOf(fa, TWO);
-        NotEquals neq = notEqualsOf(fa, TWO);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule((Or) Predicates.combineOr(asList(eq, range, neq, gt)));
-        assertEquals(TRUE, exp);
-    }
-
-    // a == 1 AND a == 2 -> nop for date/time fields
-    public void testPropagateEquals_ignoreDateTimeFields() {
-        FieldAttribute fa = TestUtils.getFieldAttribute("a", DataType.DATETIME);
-        Equals eq1 = equalsOf(fa, ONE);
-        Equals eq2 = equalsOf(fa, TWO);
-        And and = new And(EMPTY, eq1, eq2);
-
-        PropagateEquals rule = new PropagateEquals();
-        Expression exp = rule.rule(and);
-        assertEquals(and, exp);
-    }
-
-    //
-    // Like / Regex
-    //
-    public void testMatchAllLikeToExist() throws Exception {
-        for (String s : asList("%", "%%", "%%%")) {
-            LikePattern pattern = new LikePattern(s, (char) 0);
-            FieldAttribute fa = getFieldAttribute();
-            Like l = new Like(EMPTY, fa, pattern);
-            Expression e = new ReplaceRegexMatch().rule(l);
-            assertEquals(IsNotNull.class, e.getClass());
-            IsNotNull inn = (IsNotNull) e;
-            assertEquals(fa, inn.field());
-        }
-    }
-
-    public void testMatchAllWildcardLikeToExist() throws Exception {
-        for (String s : asList("*", "**", "***")) {
-            WildcardPattern pattern = new WildcardPattern(s);
-            FieldAttribute fa = getFieldAttribute();
-            WildcardLike l = new WildcardLike(EMPTY, fa, pattern);
-            Expression e = new ReplaceRegexMatch().rule(l);
-            assertEquals(IsNotNull.class, e.getClass());
-            IsNotNull inn = (IsNotNull) e;
-            assertEquals(fa, inn.field());
-        }
-    }
-
-    public void testMatchAllRLikeToExist() throws Exception {
-        RLikePattern pattern = new RLikePattern(".*");
-        FieldAttribute fa = getFieldAttribute();
-        RLike l = new RLike(EMPTY, fa, pattern);
-        Expression e = new ReplaceRegexMatch().rule(l);
-        assertEquals(IsNotNull.class, e.getClass());
-        IsNotNull inn = (IsNotNull) e;
-        assertEquals(fa, inn.field());
-    }
-
-    public void testExactMatchLike() throws Exception {
-        for (String s : asList("ab", "ab0%", "ab0_c")) {
-            LikePattern pattern = new LikePattern(s, '0');
-            FieldAttribute fa = getFieldAttribute();
-            Like l = new Like(EMPTY, fa, pattern);
-            Expression e = new ReplaceRegexMatch().rule(l);
-            assertEquals(Equals.class, e.getClass());
-            Equals eq = (Equals) e;
-            assertEquals(fa, eq.left());
-            assertEquals(s.replace("0", StringUtils.EMPTY), eq.right().fold());
-        }
-    }
-
-    public void testExactMatchWildcardLike() throws Exception {
-        String s = "ab";
-        WildcardPattern pattern = new WildcardPattern(s);
-        FieldAttribute fa = getFieldAttribute();
-        WildcardLike l = new WildcardLike(EMPTY, fa, pattern);
-        Expression e = new ReplaceRegexMatch().rule(l);
-        assertEquals(Equals.class, e.getClass());
-        Equals eq = (Equals) e;
-        assertEquals(fa, eq.left());
-        assertEquals(s, eq.right().fold());
-    }
-
-    public void testExactMatchRLike() throws Exception {
-        RLikePattern pattern = new RLikePattern("abc");
-        FieldAttribute fa = getFieldAttribute();
-        RLike l = new RLike(EMPTY, fa, pattern);
-        Expression e = new ReplaceRegexMatch().rule(l);
-        assertEquals(Equals.class, e.getClass());
-        Equals eq = (Equals) e;
-        assertEquals(fa, eq.left());
-        assertEquals("abc", eq.right().fold());
-    }
-
-    //
-    // CombineDisjunction in Equals
-    //
-    public void testTwoEqualsWithOr() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(In.class, e.getClass());
-        In in = (In) e;
-        assertEquals(fa, in.value());
-        assertThat(in.list(), contains(ONE, TWO));
-    }
-
-    public void testTwoEqualsWithSameValue() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, ONE));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(Equals.class, e.getClass());
-        Equals eq = (Equals) e;
-        assertEquals(fa, eq.left());
-        assertEquals(ONE, eq.right());
-    }
-
-    public void testOneEqualsOneIn() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, singletonList(TWO)));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(In.class, e.getClass());
-        In in = (In) e;
-        assertEquals(fa, in.value());
-        assertThat(in.list(), contains(ONE, TWO));
-    }
-
-    public void testOneEqualsOneInWithSameValue() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, asList(ONE, TWO)));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(In.class, e.getClass());
-        In in = (In) e;
-        assertEquals(fa, in.value());
-        assertThat(in.list(), contains(ONE, TWO));
-    }
-
-    public void testSingleValueInToEquals() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        Equals equals = equalsOf(fa, ONE);
-        Or or = new Or(EMPTY, equals, new In(EMPTY, fa, singletonList(ONE)));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(equals, e);
-    }
-
-    public void testEqualsBehindAnd() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        And and = new And(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO));
-        Filter dummy = new Filter(EMPTY, relation(), and);
-        LogicalPlan transformed = new CombineDisjunctionsToIn().apply(dummy);
-        assertSame(dummy, transformed);
-        assertEquals(and, ((Filter) transformed).condition());
-    }
-
-    public void testTwoEqualsDifferentFields() throws Exception {
-        FieldAttribute fieldOne = TestUtils.getFieldAttribute("ONE");
-        FieldAttribute fieldTwo = TestUtils.getFieldAttribute("TWO");
-
-        Or or = new Or(EMPTY, equalsOf(fieldOne, ONE), equalsOf(fieldTwo, TWO));
-        Expression e = new CombineDisjunctionsToIn().rule(or);
-        assertEquals(or, e);
-    }
-
-    public void testMultipleIn() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or firstOr = new Or(EMPTY, new In(EMPTY, fa, singletonList(ONE)), new In(EMPTY, fa, singletonList(TWO)));
-        Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, singletonList(THREE)));
-        Expression e = new CombineDisjunctionsToIn().rule(secondOr);
-        assertEquals(In.class, e.getClass());
-        In in = (In) e;
-        assertEquals(fa, in.value());
-        assertThat(in.list(), contains(ONE, TWO, THREE));
-    }
-
-    public void testOrWithNonCombinableExpressions() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or firstOr = new Or(EMPTY, new In(EMPTY, fa, singletonList(ONE)), lessThanOf(fa, TWO));
-        Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, singletonList(THREE)));
-        Expression e = new CombineDisjunctionsToIn().rule(secondOr);
-        assertEquals(Or.class, e.getClass());
-        Or or = (Or) e;
-        assertEquals(or.left(), firstOr.right());
-        assertEquals(In.class, or.right().getClass());
-        In in = (In) or.right();
-        assertEquals(fa, in.value());
-        assertThat(in.list(), contains(ONE, THREE));
-    }
-
-    // Null folding
-
-    public void testNullFoldingIsNull() {
-        FoldNull foldNull = new FoldNull();
-        assertEquals(true, foldNull.rule(new IsNull(EMPTY, NULL)).fold());
-        assertEquals(false, foldNull.rule(new IsNull(EMPTY, TRUE)).fold());
-    }
-
-    public void testGenericNullableExpression() {
-        FoldNull rule = new FoldNull();
-        // arithmetic
-        assertNullLiteral(rule.rule(new Add(EMPTY, getFieldAttribute(), NULL)));
-        // comparison
-        assertNullLiteral(rule.rule(greaterThanOf(getFieldAttribute(), NULL)));
-        // regex
-        assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, new RLikePattern("123"))));
-    }
-
-    public void testNullFoldingDoesNotApplyOnLogicalExpressions() {
-        FoldNull rule = new FoldNull();
-
-        Or or = new Or(EMPTY, NULL, TRUE);
-        assertEquals(or, rule.rule(or));
-        or = new Or(EMPTY, NULL, NULL);
-        assertEquals(or, rule.rule(or));
-
-        And and = new And(EMPTY, NULL, TRUE);
-        assertEquals(and, rule.rule(and));
-        and = new And(EMPTY, NULL, NULL);
-        assertEquals(and, rule.rule(and));
-    }
-
-    //
-    // Propagate nullability (IS NULL / IS NOT NULL)
-    //
-
-    // a IS NULL AND a IS NOT NULL => false
-    public void testIsNullAndNotNull() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        And and = new And(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa));
-        assertEquals(FALSE, new PropagateNullable().rule(and));
-    }
-
-    // a IS NULL AND b IS NOT NULL AND c IS NULL AND d IS NOT NULL AND e IS NULL AND a IS NOT NULL => false
-    public void testIsNullAndNotNullMultiField() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        And andOne = new And(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, getFieldAttribute()));
-        And andTwo = new And(EMPTY, new IsNull(EMPTY, getFieldAttribute()), new IsNotNull(EMPTY, getFieldAttribute()));
-        And andThree = new And(EMPTY, new IsNull(EMPTY, getFieldAttribute()), new IsNotNull(EMPTY, fa));
-
-        And and = new And(EMPTY, andOne, new And(EMPTY, andThree, andTwo));
-
-        assertEquals(FALSE, new PropagateNullable().rule(and));
-    }
-
-    // a IS NULL AND a > 1 => a IS NULL AND false
-    public void testIsNullAndComparison() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-        IsNull isNull = new IsNull(EMPTY, fa);
-
-        And and = new And(EMPTY, isNull, greaterThanOf(fa, ONE));
-        assertEquals(new And(EMPTY, isNull, nullOf(BOOLEAN)), new PropagateNullable().rule(and));
-    }
-
-    // a IS NULL AND b < 1 AND c < 1 AND a < 1 => a IS NULL AND b < 1 AND c < 1 => a IS NULL AND b < 1 AND c < 1
-    public void testIsNullAndMultipleComparison() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-        IsNull isNull = new IsNull(EMPTY, fa);
-
-        And nestedAnd = new And(
-            EMPTY,
-            lessThanOf(TestUtils.getFieldAttribute("b"), ONE),
-            lessThanOf(TestUtils.getFieldAttribute("c"), ONE)
-        );
-        And and = new And(EMPTY, isNull, nestedAnd);
-        And top = new And(EMPTY, and, lessThanOf(fa, ONE));
-
-        Expression optimized = new PropagateNullable().rule(top);
-        Expression expected = new And(EMPTY, and, nullOf(BOOLEAN));
-        assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized));
-    }
-
-    // ((a+1)/2) > 1 AND a + 2 AND a IS NULL AND b < 3 => NULL AND NULL AND a IS NULL AND b < 3
-    public void testIsNullAndDeeplyNestedExpression() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-        IsNull isNull = new IsNull(EMPTY, fa);
-
-        Expression nullified = new And(
-            EMPTY,
-            greaterThanOf(new Div(EMPTY, new Add(EMPTY, fa, ONE), TWO), ONE),
-            greaterThanOf(new Add(EMPTY, fa, TWO), ONE)
-        );
-        Expression kept = new And(EMPTY, isNull, lessThanOf(TestUtils.getFieldAttribute("b"), THREE));
-        And and = new And(EMPTY, nullified, kept);
-
-        Expression optimized = new PropagateNullable().rule(and);
-        Expression expected = new And(EMPTY, new And(EMPTY, nullOf(BOOLEAN), nullOf(BOOLEAN)), kept);
-
-        assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized));
-    }
-
-    // a IS NULL OR a IS NOT NULL => no change
-    // a IS NULL OR a > 1 => no change
-    public void testIsNullInDisjunction() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-
-        Or or = new Or(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa));
-        Filter dummy = new Filter(EMPTY, relation(), or);
-        LogicalPlan transformed = new PropagateNullable().apply(dummy);
-        assertSame(dummy, transformed);
-        assertEquals(or, ((Filter) transformed).condition());
-
-        or = new Or(EMPTY, new IsNull(EMPTY, fa), greaterThanOf(fa, ONE));
-        dummy = new Filter(EMPTY, relation(), or);
-        transformed = new PropagateNullable().apply(dummy);
-        assertSame(dummy, transformed);
-        assertEquals(or, ((Filter) transformed).condition());
-    }
-
-    // a + 1 AND (a IS NULL OR a > 3) => no change
-    public void testIsNullDisjunction() throws Exception {
-        FieldAttribute fa = getFieldAttribute();
-        IsNull isNull = new IsNull(EMPTY, fa);
-
-        Or or = new Or(EMPTY, isNull, greaterThanOf(fa, THREE));
-        And and = new And(EMPTY, new Add(EMPTY, fa, ONE), or);
-
-        assertEquals(and, new PropagateNullable().rule(and));
-    }
-
-    public void testIsNotNullOnIsNullField() {
-        EsRelation relation = relation();
-        var fieldA = TestUtils.getFieldAttribute("a");
-        Expression inn = isNotNull(fieldA);
-        Filter f = new Filter(EMPTY, relation, inn);
-
-        assertEquals(f, new OptimizerRules.InferIsNotNull().apply(f));
-    }
-
-    public void testIsNotNullOnOperatorWithOneField() {
-        EsRelation relation = relation();
-        var fieldA = TestUtils.getFieldAttribute("a");
-        Expression inn = isNotNull(new Add(EMPTY, fieldA, ONE));
-        Filter f = new Filter(EMPTY, relation, inn);
-        Filter expected = new Filter(EMPTY, relation, new And(EMPTY, isNotNull(fieldA), inn));
-
-        assertEquals(expected, new OptimizerRules.InferIsNotNull().apply(f));
-    }
-
-    public void testIsNotNullOnOperatorWithTwoFields() {
-        EsRelation relation = relation();
-        var fieldA = TestUtils.getFieldAttribute("a");
-        var fieldB = TestUtils.getFieldAttribute("b");
-        Expression inn = isNotNull(new Add(EMPTY, fieldA, fieldB));
-        Filter f = new Filter(EMPTY, relation, inn);
-        Filter expected = new Filter(EMPTY, relation, new And(EMPTY, new And(EMPTY, isNotNull(fieldA), isNotNull(fieldB)), inn));
-
-        assertEquals(expected, new OptimizerRules.InferIsNotNull().apply(f));
-    }
-
-    public void testIsNotNullOnFunctionWithTwoField() {}
-
-    private IsNotNull isNotNull(Expression field) {
-        return new IsNotNull(EMPTY, field);
-    }
-
-    private IsNull isNull(Expression field) {
-        return new IsNull(EMPTY, field);
-    }
-
-    private Literal nullOf(DataType dataType) {
-        return new Literal(Source.EMPTY, null, dataType);
-    }
-
-    private void assertNullLiteral(Expression expression) {
-        assertEquals(Literal.class, expression.getClass());
-        assertNull(expression.fold());
-    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
index 62a4a25445ac7..b5a288408809f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
@@ -32,7 +32,9 @@
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.StringPattern;
 import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules;
 import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.ConstantFolding;
 import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.LiteralsOnTheRight;
@@ -1293,7 +1295,28 @@ protected LogicalPlan rule(LogicalPlan plan, LogicalOptimizerContext context) {
         }
     }
 
-    public static class ReplaceRegexMatch extends OptimizerRules.ReplaceRegexMatch {
+    public static class ReplaceRegexMatch extends org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.OptimizerExpressionRule<
+        RegexMatch<?>> {
+
+        ReplaceRegexMatch() {
+            super(org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.TransformDirection.DOWN);
+        }
+
+        @Override
+        public Expression rule(RegexMatch<?> regexMatch) {
+            Expression e = regexMatch;
+            StringPattern pattern = regexMatch.pattern();
+            if (pattern.matchesAll()) {
+                e = new IsNotNull(e.source(), regexMatch.field());
+            } else {
+                String match = pattern.exactMatch();
+                if (match != null) {
+                    Literal literal = new Literal(regexMatch.source(), match, DataType.KEYWORD);
+                    e = regexToEquals(regexMatch, literal);
+                }
+            }
+            return e;
+        }
 
         protected Expression regexToEquals(RegexMatch<?> regexMatch, Literal literal) {
             return new Equals(regexMatch.source(), regexMatch.field(), literal);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
index fdddbfc837a51..2041c08acbca6 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRules.java
@@ -294,12 +294,6 @@ protected Expression rule(BinaryComparison bc) {
      * When encountering a different Equals, non-containing {@link Range} or {@link BinaryComparison}, the conjunction becomes false.
      * When encountering a containing {@link Range}, {@link BinaryComparison} or {@link NotEquals}, these get eliminated by the equality.
      *
-     * Since this rule can eliminate Ranges and BinaryComparisons, it should be applied before
-     * {@link org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.CombineBinaryComparisons}.
-     *
-     * This rule doesn't perform any promotion of {@link BinaryComparison}s, that is handled by
-     * {@link org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.CombineBinaryComparisons} on purpose as the resulting Range might
-     * be foldable (which is picked by the folding rule on the next run).
      */
     public static final class PropagateEquals extends org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.OptimizerExpressionRule<
         BinaryLogic> {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java
index 6ca1a638fa0ee..b5400237bfeef 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java
@@ -9,19 +9,40 @@
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.TestUtils;
+import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Expressions;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
+import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.Like;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.LikePattern;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
+import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.ConstantFolding;
+import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.FoldNull;
+import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.PropagateNullable;
 import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.StringUtils;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
@@ -29,16 +50,21 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
+import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer.ReplaceRegexMatch;
 
 import java.util.List;
 
 import static java.util.Arrays.asList;
+import static org.elasticsearch.xpack.esql.core.TestUtils.nullEqualsOf;
+import static org.elasticsearch.xpack.esql.core.TestUtils.of;
 import static org.elasticsearch.xpack.esql.core.TestUtils.rangeOf;
 import static org.elasticsearch.xpack.esql.core.TestUtils.relation;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL;
 import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
+import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
 import static org.hamcrest.Matchers.contains;
 
 public class OptimizerRulesTests extends ESTestCase {
@@ -47,6 +73,9 @@ public class OptimizerRulesTests extends ESTestCase {
     private static final Literal THREE = new Literal(Source.EMPTY, 3, DataType.INTEGER);
     private static final Literal FOUR = new Literal(Source.EMPTY, 4, DataType.INTEGER);
     private static final Literal FIVE = new Literal(Source.EMPTY, 5, DataType.INTEGER);
+    private static final Literal SIX = new Literal(Source.EMPTY, 6, DataType.INTEGER);
+    private static final Expression DUMMY_EXPRESSION =
+        new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 0);
 
     private static Equals equalsOf(Expression left, Expression right) {
         return new Equals(EMPTY, left, right, null);
@@ -76,6 +105,82 @@ private static FieldAttribute getFieldAttribute() {
         return TestUtils.getFieldAttribute("a");
     }
 
+    //
+    // Constant folding
+    //
+
+    public void testConstantFolding() {
+        Expression exp = new Add(EMPTY, TWO, THREE);
+
+        assertTrue(exp.foldable());
+        Expression result = new ConstantFolding().rule(exp);
+        assertTrue(result instanceof Literal);
+        assertEquals(5, ((Literal) result).value());
+
+        // check now with an alias
+        result = new ConstantFolding().rule(new Alias(EMPTY, "a", exp));
+        assertEquals("a", Expressions.name(result));
+        assertEquals(Alias.class, result.getClass());
+    }
+
+    public void testConstantFoldingBinaryComparison() {
+        assertEquals(FALSE, new ConstantFolding().rule(greaterThanOf(TWO, THREE)).canonical());
+        assertEquals(FALSE, new ConstantFolding().rule(greaterThanOrEqualOf(TWO, THREE)).canonical());
+        assertEquals(FALSE, new ConstantFolding().rule(equalsOf(TWO, THREE)).canonical());
+        assertEquals(FALSE, new ConstantFolding().rule(nullEqualsOf(TWO, THREE)).canonical());
+        assertEquals(FALSE, new ConstantFolding().rule(nullEqualsOf(TWO, NULL)).canonical());
+        assertEquals(TRUE, new ConstantFolding().rule(notEqualsOf(TWO, THREE)).canonical());
+        assertEquals(TRUE, new ConstantFolding().rule(lessThanOrEqualOf(TWO, THREE)).canonical());
+        assertEquals(TRUE, new ConstantFolding().rule(lessThanOf(TWO, THREE)).canonical());
+    }
+
+    public void testConstantFoldingBinaryLogic() {
+        assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, greaterThanOf(TWO, THREE), TRUE)).canonical());
+        assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, greaterThanOrEqualOf(TWO, THREE), TRUE)).canonical());
+    }
+
+    public void testConstantFoldingBinaryLogic_WithNullHandling() {
+        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, NULL, TRUE)).canonical().nullable());
+        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, TRUE, NULL)).canonical().nullable());
+        assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, NULL, FALSE)).canonical());
+        assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, FALSE, NULL)).canonical());
+        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, NULL, NULL)).canonical().nullable());
+
+        assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, TRUE)).canonical());
+        assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, TRUE, NULL)).canonical());
+        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, FALSE)).canonical().nullable());
+        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, FALSE, NULL)).canonical().nullable());
+        assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, NULL)).canonical().nullable());
+    }
+
+    public void testConstantFoldingRange() {
+        assertEquals(true, new ConstantFolding().rule(rangeOf(FIVE, FIVE, true, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold());
+        assertEquals(false, new ConstantFolding().rule(rangeOf(FIVE, FIVE, false, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold());
+    }
+
+    public void testConstantNot() {
+        assertEquals(FALSE, new ConstantFolding().rule(new Not(EMPTY, TRUE)));
+        assertEquals(TRUE, new ConstantFolding().rule(new Not(EMPTY, FALSE)));
+    }
+
+    public void testConstantFoldingLikes() {
+        assertEquals(TRUE, new ConstantFolding().rule(new Like(EMPTY, of("test_emp"), new LikePattern("test%", (char) 0))).canonical());
+        assertEquals(TRUE, new ConstantFolding().rule(new WildcardLike(EMPTY, of("test_emp"), new WildcardPattern("test*"))).canonical());
+        assertEquals(TRUE, new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical());
+    }
+
+    public void testArithmeticFolding() {
+        assertEquals(10, foldOperator(new Add(EMPTY, new Literal(EMPTY, 7, DataType.INTEGER), THREE)));
+        assertEquals(4, foldOperator(new Sub(EMPTY, new Literal(EMPTY, 7, DataType.INTEGER), THREE)));
+        assertEquals(21, foldOperator(new Mul(EMPTY, new Literal(EMPTY, 7, DataType.INTEGER), THREE)));
+        assertEquals(2, foldOperator(new Div(EMPTY, new Literal(EMPTY, 7, DataType.INTEGER), THREE)));
+        assertEquals(1, foldOperator(new Mod(EMPTY, new Literal(EMPTY, 7, DataType.INTEGER), THREE)));
+    }
+
+    private static Object foldOperator(BinaryOperator<?, ?, ?, ?> b) {
+        return ((Literal) new ConstantFolding().rule(b)).value();
+    }
+
     //
     // CombineDisjunction in Equals
     //
@@ -500,4 +605,282 @@ public void testEliminateRangeByEqualsInInterval() {
         Expression exp = rule.rule(new And(EMPTY, eq1, r));
         assertEquals(eq1, exp);
     }
+    //
+    // Null folding
+
+    public void testNullFoldingIsNull() {
+        FoldNull foldNull = new FoldNull();
+        assertEquals(true, foldNull.rule(new IsNull(EMPTY, NULL)).fold());
+        assertEquals(false, foldNull.rule(new IsNull(EMPTY, TRUE)).fold());
+    }
+
+    public void testGenericNullableExpression() {
+        FoldNull rule = new FoldNull();
+        // arithmetic
+        assertNullLiteral(rule.rule(new Add(EMPTY, getFieldAttribute(), NULL)));
+        // comparison
+        assertNullLiteral(rule.rule(greaterThanOf(getFieldAttribute(), NULL)));
+        // regex
+        assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, new RLikePattern("123"))));
+    }
+
+    public void testNullFoldingDoesNotApplyOnLogicalExpressions() {
+        org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.FoldNull rule =
+            new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.FoldNull();
+
+        Or or = new Or(EMPTY, NULL, TRUE);
+        assertEquals(or, rule.rule(or));
+        or = new Or(EMPTY, NULL, NULL);
+        assertEquals(or, rule.rule(or));
+
+        And and = new And(EMPTY, NULL, TRUE);
+        assertEquals(and, rule.rule(and));
+        and = new And(EMPTY, NULL, NULL);
+        assertEquals(and, rule.rule(and));
+    }
+
+    //
+    // Propagate nullability (IS NULL / IS NOT NULL)
+    //
+
+    // a IS NULL AND a IS NOT NULL => false
+    public void testIsNullAndNotNull() {
+        FieldAttribute fa = getFieldAttribute();
+
+        And and = new And(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa));
+        assertEquals(FALSE, new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.PropagateNullable().rule(and));
+    }
+
+    // a IS NULL AND b IS NOT NULL AND c IS NULL AND d IS NOT NULL AND e IS NULL AND a IS NOT NULL => false
+    public void testIsNullAndNotNullMultiField() {
+        FieldAttribute fa = getFieldAttribute();
+
+        And andOne = new And(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, getFieldAttribute()));
+        And andTwo = new And(EMPTY, new IsNull(EMPTY, getFieldAttribute()), new IsNotNull(EMPTY, getFieldAttribute()));
+        And andThree = new And(EMPTY, new IsNull(EMPTY, getFieldAttribute()), new IsNotNull(EMPTY, fa));
+
+        And and = new And(EMPTY, andOne, new And(EMPTY, andThree, andTwo));
+
+        assertEquals(FALSE, new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.PropagateNullable().rule(and));
+    }
+
+    // a IS NULL AND a > 1 => a IS NULL AND false
+    public void testIsNullAndComparison() {
+        FieldAttribute fa = getFieldAttribute();
+        IsNull isNull = new IsNull(EMPTY, fa);
+
+        And and = new And(EMPTY, isNull, greaterThanOf(fa, ONE));
+        assertEquals(new And(EMPTY, isNull, nullOf(BOOLEAN)), new PropagateNullable().rule(and));
+    }
+
+    // a IS NULL AND b < 1 AND c < 1 AND a < 1 => a IS NULL AND b < 1 AND c < 1 => a IS NULL AND b < 1 AND c < 1
+    public void testIsNullAndMultipleComparison() {
+        FieldAttribute fa = getFieldAttribute();
+        IsNull isNull = new IsNull(EMPTY, fa);
+
+        And nestedAnd = new And(
+            EMPTY,
+            lessThanOf(TestUtils.getFieldAttribute("b"), ONE),
+            lessThanOf(TestUtils.getFieldAttribute("c"), ONE)
+        );
+        And and = new And(EMPTY, isNull, nestedAnd);
+        And top = new And(EMPTY, and, lessThanOf(fa, ONE));
+
+        Expression optimized = new PropagateNullable().rule(top);
+        Expression expected = new And(EMPTY, and, nullOf(BOOLEAN));
+        assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized));
+    }
+
+    // ((a+1)/2) > 1 AND a + 2 AND a IS NULL AND b < 3 => NULL AND NULL AND a IS NULL AND b < 3
+    public void testIsNullAndDeeplyNestedExpression() {
+        FieldAttribute fa = getFieldAttribute();
+        IsNull isNull = new IsNull(EMPTY, fa);
+
+        Expression nullified = new And(
+            EMPTY,
+            greaterThanOf(new Div(EMPTY, new Add(EMPTY, fa, ONE), TWO), ONE),
+            greaterThanOf(new Add(EMPTY, fa, TWO), ONE)
+        );
+        Expression kept = new And(EMPTY, isNull, lessThanOf(TestUtils.getFieldAttribute("b"), THREE));
+        And and = new And(EMPTY, nullified, kept);
+
+        Expression optimized = new PropagateNullable().rule(and);
+        Expression expected = new And(EMPTY, new And(EMPTY, nullOf(BOOLEAN), nullOf(BOOLEAN)), kept);
+
+        assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized));
+    }
+
+    // a IS NULL OR a IS NOT NULL => no change
+    // a IS NULL OR a > 1 => no change
+    public void testIsNullInDisjunction() {
+        FieldAttribute fa = getFieldAttribute();
+
+        Or or = new Or(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa));
+        Filter dummy = new Filter(EMPTY, relation(), or);
+        LogicalPlan transformed = new PropagateNullable().apply(dummy);
+        assertSame(dummy, transformed);
+        assertEquals(or, ((Filter) transformed).condition());
+
+        or = new Or(EMPTY, new IsNull(EMPTY, fa), greaterThanOf(fa, ONE));
+        dummy = new Filter(EMPTY, relation(), or);
+        transformed = new PropagateNullable().apply(dummy);
+        assertSame(dummy, transformed);
+        assertEquals(or, ((Filter) transformed).condition());
+    }
+
+    // a + 1 AND (a IS NULL OR a > 3) => no change
+    public void testIsNullDisjunction() {
+        FieldAttribute fa = getFieldAttribute();
+        IsNull isNull = new IsNull(EMPTY, fa);
+
+        Or or = new Or(EMPTY, isNull, greaterThanOf(fa, THREE));
+        And and = new And(EMPTY, new Add(EMPTY, fa, ONE), or);
+
+        assertEquals(and, new PropagateNullable().rule(and));
+    }
+
+    //
+    // Like / Regex
+    //
+    public void testMatchAllLikeToExist() {
+        for (String s : asList("%", "%%", "%%%")) {
+            LikePattern pattern = new LikePattern(s, (char) 0);
+            FieldAttribute fa = getFieldAttribute();
+            Like l = new Like(EMPTY, fa, pattern);
+            Expression e = new ReplaceRegexMatch().rule(l);
+            assertEquals(IsNotNull.class, e.getClass());
+            IsNotNull inn = (IsNotNull) e;
+            assertEquals(fa, inn.field());
+        }
+    }
+
+    public void testMatchAllWildcardLikeToExist() {
+        for (String s : asList("*", "**", "***")) {
+            WildcardPattern pattern = new WildcardPattern(s);
+            FieldAttribute fa = getFieldAttribute();
+            WildcardLike l = new WildcardLike(EMPTY, fa, pattern);
+            Expression e = new ReplaceRegexMatch().rule(l);
+            assertEquals(IsNotNull.class, e.getClass());
+            IsNotNull inn = (IsNotNull) e;
+            assertEquals(fa, inn.field());
+        }
+    }
+
+    public void testMatchAllRLikeToExist() {
+        RLikePattern pattern = new RLikePattern(".*");
+        FieldAttribute fa = getFieldAttribute();
+        RLike l = new RLike(EMPTY, fa, pattern);
+        Expression e = new ReplaceRegexMatch().rule(l);
+        assertEquals(IsNotNull.class, e.getClass());
+        IsNotNull inn = (IsNotNull) e;
+        assertEquals(fa, inn.field());
+    }
+
+    public void testExactMatchLike() {
+        for (String s : asList("ab", "ab0%", "ab0_c")) {
+            LikePattern pattern = new LikePattern(s, '0');
+            FieldAttribute fa = getFieldAttribute();
+            Like l = new Like(EMPTY, fa, pattern);
+            Expression e = new ReplaceRegexMatch().rule(l);
+            assertEquals(Equals.class, e.getClass());
+            Equals eq = (Equals) e;
+            assertEquals(fa, eq.left());
+            assertEquals(s.replace("0", StringUtils.EMPTY), eq.right().fold());
+        }
+    }
+
+    public void testExactMatchWildcardLike() {
+        String s = "ab";
+        WildcardPattern pattern = new WildcardPattern(s);
+        FieldAttribute fa = getFieldAttribute();
+        WildcardLike l = new WildcardLike(EMPTY, fa, pattern);
+        Expression e = new ReplaceRegexMatch().rule(l);
+        assertEquals(Equals.class, e.getClass());
+        Equals eq = (Equals) e;
+        assertEquals(fa, eq.left());
+        assertEquals(s, eq.right().fold());
+    }
+
+    public void testExactMatchRLike() {
+        RLikePattern pattern = new RLikePattern("abc");
+        FieldAttribute fa = getFieldAttribute();
+        RLike l = new RLike(EMPTY, fa, pattern);
+        Expression e = new ReplaceRegexMatch().rule(l);
+        assertEquals(Equals.class, e.getClass());
+        Equals eq = (Equals) e;
+        assertEquals(fa, eq.left());
+        assertEquals("abc", eq.right().fold());
+    }
+
+    private void assertNullLiteral(Expression expression) {
+        assertEquals(Literal.class, expression.getClass());
+        assertNull(expression.fold());
+    }
+
+    private IsNotNull isNotNull(Expression field) {
+        return new IsNotNull(EMPTY, field);
+    }
+
+    private IsNull isNull(Expression field) {
+        return new IsNull(EMPTY, field);
+    }
+
+    private Literal nullOf(DataType dataType) {
+        return new Literal(Source.EMPTY, null, dataType);
+    }
+    //
+    // Logical simplifications
+    //
+
+    public void testLiteralsOnTheRight() {
+        Alias a = new Alias(EMPTY, "a", new Literal(EMPTY, 10, INTEGER));
+        Expression result = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.LiteralsOnTheRight().rule(equalsOf(FIVE, a));
+        assertTrue(result instanceof Equals);
+        Equals eq = (Equals) result;
+        assertEquals(a, eq.left());
+        assertEquals(FIVE, eq.right());
+
+        // Note: Null Equals test removed here
+    }
+
+    public void testBoolSimplifyOr() {
+        org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BooleanSimplification simplification =
+            new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BooleanSimplification();
+
+        assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, TRUE)));
+        assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, DUMMY_EXPRESSION)));
+        assertEquals(TRUE, simplification.rule(new Or(EMPTY, DUMMY_EXPRESSION, TRUE)));
+
+        assertEquals(FALSE, simplification.rule(new Or(EMPTY, FALSE, FALSE)));
+        assertEquals(DUMMY_EXPRESSION, simplification.rule(new Or(EMPTY, FALSE, DUMMY_EXPRESSION)));
+        assertEquals(DUMMY_EXPRESSION, simplification.rule(new Or(EMPTY, DUMMY_EXPRESSION, FALSE)));
+    }
+
+    public void testBoolSimplifyAnd() {
+        org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BooleanSimplification simplification =
+            new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BooleanSimplification();
+
+        assertEquals(TRUE, simplification.rule(new And(EMPTY, TRUE, TRUE)));
+        assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, TRUE, DUMMY_EXPRESSION)));
+        assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, DUMMY_EXPRESSION, TRUE)));
+
+        assertEquals(FALSE, simplification.rule(new And(EMPTY, FALSE, FALSE)));
+        assertEquals(FALSE, simplification.rule(new And(EMPTY, FALSE, DUMMY_EXPRESSION)));
+        assertEquals(FALSE, simplification.rule(new And(EMPTY, DUMMY_EXPRESSION, FALSE)));
+    }
+
+    public void testBoolCommonFactorExtraction() {
+        org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BooleanSimplification simplification =
+            new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BooleanSimplification();
+
+        Expression a1 = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 1);
+        Expression a2 = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 1);
+        Expression b = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 2);
+        Expression c = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 3);
+
+        Or actual = new Or(EMPTY, new And(EMPTY, a1, b), new And(EMPTY, a2, c));
+        And expected = new And(EMPTY, a1, new Or(EMPTY, b, c));
+
+        assertEquals(expected, simplification.rule(actual));
+    }
 }

From aab2db57170473536e8b407f7718421bcfbaf0f3 Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Wed, 5 Jun 2024 16:14:35 +0100
Subject: [PATCH 171/208] [ML] Fix serialising inference delete response 
 (#109384)

---
 docs/changelog/109384.yaml                         |  5 +++++
 .../action/DeleteInferenceEndpointAction.java      |  2 +-
 .../TransportDeleteInferenceEndpointAction.java    | 14 +++++++++-----
 3 files changed, 15 insertions(+), 6 deletions(-)
 create mode 100644 docs/changelog/109384.yaml

diff --git a/docs/changelog/109384.yaml b/docs/changelog/109384.yaml
new file mode 100644
index 0000000000000..303da23d57d8e
--- /dev/null
+++ b/docs/changelog/109384.yaml
@@ -0,0 +1,5 @@
+pr: 109384
+summary: Fix serialising inference delete response
+area: Machine Learning
+type: bug
+issues: []
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java
index be4c87195e9d1..19542ef466156 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/DeleteInferenceEndpointAction.java
@@ -20,7 +20,7 @@
 import java.util.Objects;
 import java.util.Set;
 
-public class DeleteInferenceEndpointAction extends ActionType<AcknowledgedResponse> {
+public class DeleteInferenceEndpointAction extends ActionType<DeleteInferenceEndpointAction.Response> {
 
     public static final DeleteInferenceEndpointAction INSTANCE = new DeleteInferenceEndpointAction();
     public static final String NAME = "cluster:admin/xpack/inference/delete";
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
index 5ebca2bc512a0..07d5e1e618578 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java
@@ -13,8 +13,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.SubscribableListener;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -36,7 +35,9 @@
 
 import java.util.Set;
 
-public class TransportDeleteInferenceEndpointAction extends AcknowledgedTransportMasterNodeAction<DeleteInferenceEndpointAction.Request> {
+public class TransportDeleteInferenceEndpointAction extends TransportMasterNodeAction<
+    DeleteInferenceEndpointAction.Request,
+    DeleteInferenceEndpointAction.Response> {
 
     private final ModelRegistry modelRegistry;
     private final InferenceServiceRegistry serviceRegistry;
@@ -60,6 +61,7 @@ public TransportDeleteInferenceEndpointAction(
             actionFilters,
             DeleteInferenceEndpointAction.Request::new,
             indexNameExpressionResolver,
+            DeleteInferenceEndpointAction.Response::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
         this.modelRegistry = modelRegistry;
@@ -71,7 +73,7 @@ protected void masterOperation(
         Task task,
         DeleteInferenceEndpointAction.Request request,
         ClusterState state,
-        ActionListener<AcknowledgedResponse> masterListener
+        ActionListener<DeleteInferenceEndpointAction.Response> masterListener
     ) {
         SubscribableListener.<ModelRegistry.UnparsedModel>newForked(modelConfigListener -> {
             // Get the model from the registry
@@ -123,7 +125,9 @@ && endpointIsReferencedInPipelines(state, request.getInferenceEndpointId(), list
             }
         })
             .addListener(
-                masterListener.delegateFailure((l3, didDeleteModel) -> masterListener.onResponse(AcknowledgedResponse.of(didDeleteModel)))
+                masterListener.delegateFailure(
+                    (l3, didDeleteModel) -> masterListener.onResponse(new DeleteInferenceEndpointAction.Response(didDeleteModel, Set.of()))
+                )
             );
     }
 

From cccac69344d2d0df2bfda2d67692ec7ba93a87b8 Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Wed, 5 Jun 2024 16:16:53 +0100
Subject: [PATCH 172/208] [ML] Use the multi node routing action for internal
 inference services (#109358)

The Elser and internal Elasticsearch inference services were calling the wrong
inference action that did not distribute the work among all available nodes, rather
work was duplicated by all nodes. This bug does not apply when the inference
ingest processor is used and is only noticeable when an inference request contains
many inputs.
---
 docs/changelog/109358.yaml                    |  5 ++
 .../org/elasticsearch/TransportVersions.java  |  1 +
 .../core/ml/action/InferModelAction.java      | 32 ++++++++-
 .../action/InferModelActionRequestTests.java  | 21 +++++-
 .../ElasticsearchInternalService.java         | 67 +++++++++++++------
 .../services/elser/ElserInternalService.java  | 37 ++++++----
 .../ElasticsearchInternalServiceTests.java    | 39 ++++++++---
 .../elser/ElserInternalServiceTests.java      | 12 ++--
 .../TransportInternalInferModelAction.java    |  1 +
 9 files changed, 163 insertions(+), 52 deletions(-)
 create mode 100644 docs/changelog/109358.yaml

diff --git a/docs/changelog/109358.yaml b/docs/changelog/109358.yaml
new file mode 100644
index 0000000000000..af47b4129d874
--- /dev/null
+++ b/docs/changelog/109358.yaml
@@ -0,0 +1,5 @@
+pr: 109358
+summary: Use the multi node routing action for internal inference services
+area: Machine Learning
+type: bug
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index 07579161a85c8..e8a33217b937d 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -183,6 +183,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_ENHANCE_DELETE_ENDPOINT = def(8_674_00_0);
     public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_EMBEDDINGS_ADDED = def(8_675_00_0);
     public static final TransportVersion ADD_MISTRAL_EMBEDDINGS_INFERENCE = def(8_676_00_0);
+    public static final TransportVersion ML_CHUNK_INFERENCE_OPTION = def(8_677_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java
index eb5f1d4f086d0..e6b580f62fdd3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java
@@ -90,6 +90,7 @@ public static Builder parseRequest(String id, XContentParser parser) {
         private final List<String> textInput;
         private boolean highPriority;
         private TrainedModelPrefixStrings.PrefixType prefixType = TrainedModelPrefixStrings.PrefixType.NONE;
+        private boolean chunked = false;
 
         /**
          * Build a request from a list of documents as maps.
@@ -197,6 +198,11 @@ public Request(StreamInput in) throws IOException {
             } else {
                 prefixType = TrainedModelPrefixStrings.PrefixType.NONE;
             }
+            if (in.getTransportVersion().onOrAfter(TransportVersions.ML_CHUNK_INFERENCE_OPTION)) {
+                chunked = in.readBoolean();
+            } else {
+                chunked = false;
+            }
         }
 
         public int numberOfDocuments() {
@@ -247,6 +253,14 @@ public TrainedModelPrefixStrings.PrefixType getPrefixType() {
             return prefixType;
         }
 
+        public void setChunked(boolean chunked) {
+            this.chunked = chunked;
+        }
+
+        public boolean isChunked() {
+            return chunked;
+        }
+
         @Override
         public ActionRequestValidationException validate() {
             return null;
@@ -271,6 +285,9 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
                 out.writeEnum(prefixType);
             }
+            if (out.getTransportVersion().onOrAfter(TransportVersions.ML_CHUNK_INFERENCE_OPTION)) {
+                out.writeBoolean(chunked);
+            }
         }
 
         @Override
@@ -285,7 +302,8 @@ public boolean equals(Object o) {
                 && Objects.equals(objectsToInfer, that.objectsToInfer)
                 && Objects.equals(textInput, that.textInput)
                 && (highPriority == that.highPriority)
-                && (prefixType == that.prefixType);
+                && (prefixType == that.prefixType)
+                && (chunked == that.chunked);
         }
 
         @Override
@@ -295,7 +313,17 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
 
         @Override
         public int hashCode() {
-            return Objects.hash(id, objectsToInfer, update, previouslyLicensed, inferenceTimeout, textInput, highPriority, prefixType);
+            return Objects.hash(
+                id,
+                objectsToInfer,
+                update,
+                previouslyLicensed,
+                inferenceTimeout,
+                textInput,
+                highPriority,
+                prefixType,
+                chunked
+            );
         }
 
         public static class Builder {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java
index 983e5d43a946d..2e4689de787b3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java
@@ -73,6 +73,7 @@ protected Request createTestInstance() {
         if (randomBoolean()) {
             request.setPrefixType(randomFrom(TrainedModelPrefixStrings.PrefixType.values()));
         }
+        request.setChunked(randomBoolean());
         return request;
     }
 
@@ -87,8 +88,9 @@ protected Request mutateInstance(Request instance) {
         var previouslyLicensed = instance.isPreviouslyLicensed();
         var timeout = instance.getInferenceTimeout();
         var prefixType = instance.getPrefixType();
+        var chunked = instance.isChunked();
 
-        int change = randomIntBetween(0, 7);
+        int change = randomIntBetween(0, 8);
         switch (change) {
             case 0:
                 modelId = modelId + "foo";
@@ -123,6 +125,9 @@ protected Request mutateInstance(Request instance) {
                 prefixType = TrainedModelPrefixStrings.PrefixType.values()[(prefixType.ordinal() + 1) % TrainedModelPrefixStrings.PrefixType
                     .values().length];
                 break;
+            case 8:
+                chunked = chunked == false;
+                break;
             default:
                 throw new IllegalStateException();
         }
@@ -130,6 +135,7 @@ protected Request mutateInstance(Request instance) {
         var r = new Request(modelId, update, objectsToInfer, textInput, timeout, previouslyLicensed);
         r.setHighPriority(highPriority);
         r.setPrefixType(prefixType);
+        r.setChunked(chunked);
         return r;
     }
 
@@ -246,6 +252,19 @@ protected Request mutateInstanceForVersion(Request instance, TransportVersion ve
             r.setHighPriority(instance.isHighPriority());
             r.setPrefixType(TrainedModelPrefixStrings.PrefixType.NONE);
             return r;
+        } else if (version.before(TransportVersions.ML_CHUNK_INFERENCE_OPTION)) {
+            var r = new Request(
+                instance.getId(),
+                adjustedUpdate,
+                instance.getObjectsToInfer(),
+                instance.getTextInput(),
+                instance.getInferenceTimeout(),
+                instance.isPreviouslyLicensed()
+            );
+            r.setHighPriority(instance.isHighPriority());
+            r.setPrefixType(instance.getPrefixType());
+            r.setChunked(false);  // r.setChunked(instance.isChunked()); for the next version
+            return r;
         }
 
         return instance;
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
index 408e3ec1ccbca..67a45ba8b1295 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java
@@ -33,13 +33,15 @@
 import org.elasticsearch.xpack.core.inference.results.RankedDocsResults;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
-import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
+import org.elasticsearch.xpack.core.ml.action.InferModelAction;
 import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction;
 import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput;
+import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings;
 import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate;
@@ -239,7 +241,7 @@ public void infer(
         if (TaskType.TEXT_EMBEDDING.equals(taskType)) {
             inferTextEmbedding(model, input, inputType, timeout, listener);
         } else if (TaskType.RERANK.equals(taskType)) {
-            inferRerank(model, query, input, timeout, taskSettings, listener);
+            inferRerank(model, query, input, inputType, timeout, taskSettings, listener);
         } else {
             throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST);
         }
@@ -247,22 +249,26 @@ public void infer(
 
     public void inferTextEmbedding(
         Model model,
-        List<String> input,
+        List<String> inputs,
         InputType inputType,
         TimeValue timeout,
         ActionListener<InferenceServiceResults> listener
     ) {
-        var request = InferTrainedModelDeploymentAction.Request.forTextInput(
+        var request = buildInferenceRequest(
             model.getConfigurations().getInferenceEntityId(),
             TextEmbeddingConfigUpdate.EMPTY_INSTANCE,
-            input,
-            timeout
+            inputs,
+            inputType,
+            timeout,
+            false
         );
 
         client.execute(
-            InferTrainedModelDeploymentAction.INSTANCE,
+            InferModelAction.INSTANCE,
             request,
-            listener.delegateFailureAndWrap((l, inferenceResult) -> l.onResponse(TextEmbeddingResults.of(inferenceResult.getResults())))
+            listener.delegateFailureAndWrap(
+                (l, inferenceResult) -> l.onResponse(TextEmbeddingResults.of(inferenceResult.getInferenceResults()))
+            )
         );
     }
 
@@ -270,16 +276,18 @@ public void inferRerank(
         Model model,
         String query,
         List<String> inputs,
+        InputType inputType,
         TimeValue timeout,
         Map<String, Object> requestTaskSettings,
         ActionListener<InferenceServiceResults> listener
     ) {
-        var config = new TextSimilarityConfigUpdate(query);
-        var request = InferTrainedModelDeploymentAction.Request.forTextInput(
+        var request = buildInferenceRequest(
             model.getConfigurations().getInferenceEntityId(),
-            config,
+            new TextSimilarityConfigUpdate(query),
             inputs,
-            timeout
+            inputType,
+            timeout,
+            false
         );
 
         var modelSettings = (CustomElandRerankTaskSettings) model.getTaskSettings();
@@ -289,10 +297,12 @@ public void inferRerank(
         Function<Integer, String> inputSupplier = returnDocs == Boolean.TRUE ? inputs::get : i -> null;
 
         client.execute(
-            InferTrainedModelDeploymentAction.INSTANCE,
+            InferModelAction.INSTANCE,
             request,
             listener.delegateFailureAndWrap(
-                (l, inferenceResult) -> l.onResponse(textSimilarityResultsToRankedDocs(inferenceResult.getResults(), inputSupplier))
+                (l, inferenceResult) -> l.onResponse(
+                    textSimilarityResultsToRankedDocs(inferenceResult.getInferenceResults(), inputSupplier)
+                )
             )
         );
     }
@@ -331,18 +341,21 @@ public void chunkedInfer(
             ? new TokenizationConfigUpdate(chunkingOptions.windowSize(), chunkingOptions.span())
             : new TokenizationConfigUpdate(null, null);
 
-        var request = InferTrainedModelDeploymentAction.Request.forTextInput(
+        var request = buildInferenceRequest(
             model.getConfigurations().getInferenceEntityId(),
             configUpdate,
             input,
-            timeout
+            inputType,
+            timeout,
+            true
         );
-        request.setChunkResults(true);
 
         client.execute(
-            InferTrainedModelDeploymentAction.INSTANCE,
+            InferModelAction.INSTANCE,
             request,
-            listener.delegateFailureAndWrap((l, inferenceResult) -> l.onResponse(translateToChunkedResults(inferenceResult.getResults())))
+            listener.delegateFailureAndWrap(
+                (l, inferenceResult) -> l.onResponse(translateToChunkedResults(inferenceResult.getInferenceResults()))
+            )
         );
     }
 
@@ -532,4 +545,20 @@ private RankedDocsResults textSimilarityResultsToRankedDocs(
         return new RankedDocsResults(rankings);
     }
 
+    public static InferModelAction.Request buildInferenceRequest(
+        String id,
+        InferenceConfigUpdate update,
+        List<String> inputs,
+        InputType inputType,
+        TimeValue timeout,
+        boolean chunk
+    ) {
+        var request = InferModelAction.Request.forTextInput(id, update, inputs, true, timeout);
+        request.setPrefixType(
+            InputType.SEARCH == inputType ? TrainedModelPrefixStrings.PrefixType.SEARCH : TrainedModelPrefixStrings.PrefixType.INGEST
+        );
+        request.setHighPriority(InputType.SEARCH == inputType);
+        request.setChunked(chunk);
+        return request;
+    }
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java
index 01829bfee5241..a19e377d59c18 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java
@@ -34,7 +34,7 @@
 import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
-import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
+import org.elasticsearch.xpack.core.ml.action.InferModelAction;
 import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction;
 import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.action.StopTrainedModelDeploymentAction;
@@ -58,6 +58,7 @@
 import static org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus.State.STARTED;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull;
 import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap;
+import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.buildInferenceRequest;
 
 public class ElserInternalService implements InferenceService {
 
@@ -259,7 +260,7 @@ public void stop(String inferenceEntityId, ActionListener<Boolean> listener) {
     public void infer(
         Model model,
         @Nullable String query,
-        List<String> input,
+        List<String> inputs,
         Map<String, Object> taskSettings,
         InputType inputType,
         TimeValue timeout,
@@ -274,16 +275,21 @@ public void infer(
             return;
         }
 
-        var request = InferTrainedModelDeploymentAction.Request.forTextInput(
+        var request = buildInferenceRequest(
             model.getConfigurations().getInferenceEntityId(),
             TextExpansionConfigUpdate.EMPTY_UPDATE,
-            input,
-            timeout
+            inputs,
+            inputType,
+            timeout,
+            false // chunk
         );
+
         client.execute(
-            InferTrainedModelDeploymentAction.INSTANCE,
+            InferModelAction.INSTANCE,
             request,
-            listener.delegateFailureAndWrap((l, inferenceResult) -> l.onResponse(SparseEmbeddingResults.of(inferenceResult.getResults())))
+            listener.delegateFailureAndWrap(
+                (l, inferenceResult) -> l.onResponse(SparseEmbeddingResults.of(inferenceResult.getInferenceResults()))
+            )
         );
     }
 
@@ -303,7 +309,7 @@ public void chunkedInfer(
     public void chunkedInfer(
         Model model,
         @Nullable String query,
-        List<String> input,
+        List<String> inputs,
         Map<String, Object> taskSettings,
         InputType inputType,
         @Nullable ChunkingOptions chunkingOptions,
@@ -321,18 +327,21 @@ public void chunkedInfer(
             ? new TokenizationConfigUpdate(chunkingOptions.windowSize(), chunkingOptions.span())
             : new TokenizationConfigUpdate(null, null);
 
-        var request = InferTrainedModelDeploymentAction.Request.forTextInput(
+        var request = buildInferenceRequest(
             model.getConfigurations().getInferenceEntityId(),
             configUpdate,
-            input,
-            timeout
+            inputs,
+            inputType,
+            timeout,
+            true // chunk
         );
-        request.setChunkResults(true);
 
         client.execute(
-            InferTrainedModelDeploymentAction.INSTANCE,
+            InferModelAction.INSTANCE,
             request,
-            listener.delegateFailureAndWrap((l, inferenceResult) -> l.onResponse(translateChunkedResults(inferenceResult.getResults())))
+            listener.delegateFailureAndWrap(
+                (l, inferenceResult) -> l.onResponse(translateChunkedResults(inferenceResult.getInferenceResults()))
+            )
         );
     }
 
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
index ea11e9d0343e3..b06f8b0027caf 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java
@@ -30,10 +30,13 @@
 import org.elasticsearch.xpack.core.inference.results.ChunkedTextEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
 import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
+import org.elasticsearch.xpack.core.ml.action.InferModelAction;
 import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
+import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings;
 import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextEmbeddingResultsTests;
 import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate;
 import org.elasticsearch.xpack.inference.services.ServiceFields;
 import org.elasticsearch.xpack.inference.services.settings.InternalServiceSettings;
@@ -468,21 +471,16 @@ public void testChunkInfer() {
         mlTrainedModelResults.add(ChunkedTextEmbeddingResultsTests.createRandomResults());
         mlTrainedModelResults.add(ChunkedTextEmbeddingResultsTests.createRandomResults());
         mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom")));
-        var response = new InferTrainedModelDeploymentAction.Response(mlTrainedModelResults);
+        var response = new InferModelAction.Response(mlTrainedModelResults, "foo", true);
 
         ThreadPool threadpool = new TestThreadPool("test");
         Client client = mock(Client.class);
         when(client.threadPool()).thenReturn(threadpool);
         doAnswer(invocationOnMock -> {
-            var listener = (ActionListener<InferTrainedModelDeploymentAction.Response>) invocationOnMock.getArguments()[2];
+            var listener = (ActionListener<InferModelAction.Response>) invocationOnMock.getArguments()[2];
             listener.onResponse(response);
             return null;
-        }).when(client)
-            .execute(
-                same(InferTrainedModelDeploymentAction.INSTANCE),
-                any(InferTrainedModelDeploymentAction.Request.class),
-                any(ActionListener.class)
-            );
+        }).when(client).execute(same(InferModelAction.INSTANCE), any(InferModelAction.Request.class), any(ActionListener.class));
 
         var model = new MultilingualE5SmallModel(
             "foo",
@@ -644,6 +642,31 @@ public void testParsePersistedConfig_Rerank() {
         }
     }
 
+    public void testBuildInferenceRequest() {
+        var id = randomAlphaOfLength(5);
+        var inputs = randomList(1, 3, () -> randomAlphaOfLength(4));
+        var inputType = randomFrom(InputType.SEARCH, InputType.INGEST);
+        var timeout = randomTimeValue();
+        var chunk = randomBoolean();
+        var request = ElasticsearchInternalService.buildInferenceRequest(
+            id,
+            TextEmbeddingConfigUpdate.EMPTY_INSTANCE,
+            inputs,
+            inputType,
+            timeout,
+            chunk
+        );
+
+        assertEquals(id, request.getId());
+        assertEquals(inputs, request.getTextInput());
+        assertEquals(
+            inputType == InputType.INGEST ? TrainedModelPrefixStrings.PrefixType.INGEST : TrainedModelPrefixStrings.PrefixType.SEARCH,
+            request.getPrefixType()
+        );
+        assertEquals(timeout, request.getInferenceTimeout());
+        assertEquals(chunk, request.isChunked());
+    }
+
     private ElasticsearchInternalService createService(Client client) {
         var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client);
         return new ElasticsearchInternalService(context);
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java
index dcbb523cceed9..2fdb208a56e1b 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.xpack.core.inference.action.InferenceAction;
 import org.elasticsearch.xpack.core.inference.results.ChunkedSparseEmbeddingResults;
 import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults;
+import org.elasticsearch.xpack.core.ml.action.InferModelAction;
 import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.results.ChunkedTextExpansionResultsTests;
 import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults;
@@ -337,21 +338,16 @@ public void testChunkInfer() {
         mlTrainedModelResults.add(ChunkedTextExpansionResultsTests.createRandomResults());
         mlTrainedModelResults.add(ChunkedTextExpansionResultsTests.createRandomResults());
         mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom")));
-        var response = new InferTrainedModelDeploymentAction.Response(mlTrainedModelResults);
+        var response = new InferModelAction.Response(mlTrainedModelResults, "foo", true);
 
         ThreadPool threadpool = new TestThreadPool("test");
         Client client = mock(Client.class);
         when(client.threadPool()).thenReturn(threadpool);
         doAnswer(invocationOnMock -> {
-            var listener = (ActionListener<InferTrainedModelDeploymentAction.Response>) invocationOnMock.getArguments()[2];
+            var listener = (ActionListener<InferModelAction.Response>) invocationOnMock.getArguments()[2];
             listener.onResponse(response);
             return null;
-        }).when(client)
-            .execute(
-                same(InferTrainedModelDeploymentAction.INSTANCE),
-                any(InferTrainedModelDeploymentAction.Request.class),
-                any(ActionListener.class)
-            );
+        }).when(client).execute(same(InferModelAction.INSTANCE), any(InferModelAction.Request.class), any(ActionListener.class));
 
         var model = new ElserInternalModel(
             "foo",
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
index d54cac9dca496..004d87d643962 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
@@ -290,6 +290,7 @@ private void inferAgainstAllocatedModel(
             deploymentRequest.setPrefixType(request.getPrefixType());
             deploymentRequest.setNodes(node.v1());
             deploymentRequest.setParentTask(parentTaskId);
+            deploymentRequest.setChunkResults(request.isChunked());
 
             startPos += node.v2();
 

From c80a32e76fd99c3f8961be6d35696c1fa8a20b0b Mon Sep 17 00:00:00 2001
From: David Kyle <david.kyle@elastic.co>
Date: Wed, 5 Jun 2024 16:19:37 +0100
Subject: [PATCH 173/208] [ML] Reset retryable index requests after failures
 (#109320)

Fixes the `autoGeneratedTimestamp should not be set externally` error
---
 docs/changelog/109320.yaml                    |  5 ++++
 .../persistence/ResultsPersisterService.java  |  7 +++--
 .../ResultsPersisterServiceTests.java         | 29 +++++++++++++++++++
 3 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100644 docs/changelog/109320.yaml

diff --git a/docs/changelog/109320.yaml b/docs/changelog/109320.yaml
new file mode 100644
index 0000000000000..84aff5b1d769d
--- /dev/null
+++ b/docs/changelog/109320.yaml
@@ -0,0 +1,5 @@
+pr: 109320
+summary: Reset retryable index requests after failures
+area: Machine Learning
+type: bug
+issues: []
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java
index 82d19f9d72273..83572b02f754d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java
@@ -325,7 +325,7 @@ private static boolean isIrrecoverable(Exception ex) {
     }
 
     @SuppressWarnings("NonAtomicOperationOnVolatileField")
-    private static class BulkRequestRewriter {
+    static class BulkRequestRewriter {
         private volatile BulkRequest bulkRequest;
 
         BulkRequestRewriter(BulkRequest initialRequest) {
@@ -533,7 +533,7 @@ public void cancel(Exception e) {
         }
     }
 
-    private static BulkRequest buildNewRequestFromFailures(BulkRequest bulkRequest, BulkResponse bulkResponse) {
+    static BulkRequest buildNewRequestFromFailures(BulkRequest bulkRequest, BulkResponse bulkResponse) {
         // If we failed, lets set the bulkRequest to be a collection of the failed requests
         BulkRequest bulkRequestOfFailures = new BulkRequest();
         Set<String> failedDocIds = Arrays.stream(bulkResponse.getItems())
@@ -542,6 +542,9 @@ private static BulkRequest buildNewRequestFromFailures(BulkRequest bulkRequest,
             .collect(Collectors.toSet());
         bulkRequest.requests().forEach(docWriteRequest -> {
             if (failedDocIds.contains(docWriteRequest.id())) {
+                if (docWriteRequest instanceof IndexRequest ir) {
+                    ir.reset();
+                }
                 bulkRequestOfFailures.add(docWriteRequest);
             }
         });
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java
index 2acf2e3da3cf6..e109f2995d215 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java
@@ -60,6 +60,7 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.any;
@@ -375,6 +376,34 @@ public void testBulkRequestRetriesMsgHandlerIsCalled() {
         assertThat(lastMessage.get(), containsString("failed to index after [1] attempts. Will attempt again"));
     }
 
+    public void testBuildNewRequestFromFailures_resetsId() {
+        var bulkRequest = new BulkRequest();
+        var indexRequestAutoGeneratedId = new IndexRequest("index-foo");
+        indexRequestAutoGeneratedId.autoGenerateId();
+        var autoGenId = indexRequestAutoGeneratedId.id();
+        var plainIndexRequest = new IndexRequest("index-foo2").id("id-set");
+
+        bulkRequest.add(indexRequestAutoGeneratedId);
+        bulkRequest.add(plainIndexRequest);
+
+        var bulkResponse = mock(BulkResponse.class);
+
+        var failed = mock(BulkItemResponse.class);
+        when(failed.isFailed()).thenReturn(Boolean.TRUE);
+        when(failed.getId()).thenReturn(autoGenId);
+
+        var sucessful = mock(BulkItemResponse.class);
+        when(sucessful.isFailed()).thenReturn(Boolean.FALSE);
+
+        when(bulkResponse.getItems()).thenReturn(new BulkItemResponse[] { failed, sucessful });
+
+        var modifiedRequestForRetry = ResultsPersisterService.buildNewRequestFromFailures(bulkRequest, bulkResponse);
+        assertThat(modifiedRequestForRetry.requests(), hasSize(1)); // only the failed item is in the new request
+        assertThat(modifiedRequestForRetry.requests().get(0), instanceOf(IndexRequest.class));
+        var ir = (IndexRequest) modifiedRequestForRetry.requests().get(0);
+        assertEquals(ir.getAutoGeneratedTimestamp(), -1L); // failed request was reset
+    }
+
     private static <Response> Stubber doAnswerWithResponses(Response response1, Response response2) {
         return doAnswer(withResponse(response1)).doAnswer(withResponse(response2));
     }

From 80a22ec04678e32e8a501930579a98920cf9df27 Mon Sep 17 00:00:00 2001
From: "Mark J. Hoy" <mark.hoy@elastic.co>
Date: Wed, 5 Jun 2024 11:23:29 -0400
Subject: [PATCH 174/208] [Inference API] Add Docs for Mistral Embedding
 Support for the Inference API (#109319)

* Initial docs for put-inference for Mistral

* adds mistral embeddings to tutorial; add changelog

* update mistral text and dimensions

* fix mistral spelling error

* fix azure AI studio; fix Mistral label

* fix auto-formatted items

* change pipeline button back to azure openai

* put proper Azure AI Studio include in

* fix missing azure-openai; fix huggingface hidden

* fix mistral tab for reindex

* re-add Mistral service settings to put inference
---
 docs/changelog/109194.yaml                    |  5 ++
 .../inference/put-inference.asciidoc          | 61 ++++++++++++++++-
 .../semantic-search-inference.asciidoc        | 64 ++++++++----------
 .../infer-api-ingest-pipeline-widget.asciidoc | 17 +++++
 .../infer-api-ingest-pipeline.asciidoc        | 26 ++++++++
 .../infer-api-mapping-widget.asciidoc         | 20 +++++-
 .../inference-api/infer-api-mapping.asciidoc  | 34 ++++++++++
 .../infer-api-reindex-widget.asciidoc         | 21 +++++-
 .../inference-api/infer-api-reindex.asciidoc  | 23 +++++++
 .../infer-api-requirements-widget.asciidoc    | 17 +++++
 .../infer-api-requirements.asciidoc           |  6 ++
 .../infer-api-search-widget.asciidoc          | 17 +++++
 .../inference-api/infer-api-search.asciidoc   | 65 +++++++++++++++++++
 .../infer-api-task-widget.asciidoc            | 17 +++++
 .../inference-api/infer-api-task.asciidoc     | 20 ++++++
 15 files changed, 372 insertions(+), 41 deletions(-)
 create mode 100644 docs/changelog/109194.yaml

diff --git a/docs/changelog/109194.yaml b/docs/changelog/109194.yaml
new file mode 100644
index 0000000000000..bf50139547f62
--- /dev/null
+++ b/docs/changelog/109194.yaml
@@ -0,0 +1,5 @@
+pr: 109194
+summary: "[Inference API] Add Mistral Embeddings Support to Inference API"
+area: Machine Learning
+type: enhancement
+issues: [ ]
diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc
index e7d66e930e81f..5060e47447f03 100644
--- a/docs/reference/inference/put-inference.asciidoc
+++ b/docs/reference/inference/put-inference.asciidoc
@@ -7,7 +7,7 @@ experimental[]
 Creates an {infer} endpoint to perform an {infer} task.
 
 IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in
-{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure OpenAI, Google AI Studio or Hugging Face.
+{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Mistral, Azure OpenAI, Google AI Studio or Hugging Face.
 For built-in models and models uploaded though Eland, the {infer} APIs offer an alternative way to use and manage trained models.
 However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the
 <<ml-df-trained-models-apis>>.
@@ -41,6 +41,7 @@ The following services are available through the {infer} API:
 * ELSER
 * Google AI Studio
 * Hugging Face
+* Mistral
 * OpenAI
 
 [discrete]
@@ -76,6 +77,7 @@ Available services:
 * `elser`: specify the `sparse_embedding` task type to use the ELSER service.
 * `googleaistudio`: specify the `completion` or `text_embeddig` task to use the Google AI Studio service.
 * `hugging_face`: specify the `text_embedding` task type to use the Hugging Face service.
+* `mistral`: specify the `text_embedding` task type to use the Mistral service.
 * `openai`: specify the `completion` or `text_embedding` task type to use the OpenAI service.
 
 
@@ -341,6 +343,41 @@ To modify this, set the `requests_per_minute` setting of this object in your ser
 }
 ----
 
+=====
++
+.`service_settings` for the `mistral` service
+[%collapsible%closed]
+=====
+
+`api_key`:::
+(Required, string)
+A valid API key for your Mistral account.
+You can find your Mistral API keys or you can create a new one
+https://console.mistral.ai/api-keys/[on the API Keys page].
+
+`model`:::
+(Required, string)
+The name of the model to use for the {infer} task.
+Refer to the https://docs.mistral.ai/getting-started/models/[Mistral models documentation]
+for the list of available text embedding models.
+
+`max_input_tokens`:::
+(Optional, integer)
+Allows you to specify the maximum number of tokens per input before chunking occurs.
+
+`rate_limit`:::
+(Optional, object)
+By default, the `mistral` service sets the number of requests allowed per minute to `240`.
+This helps to minimize the number of rate limit errors returned from the Mistral API.
+To modify this, set the `requests_per_minute` setting of this object in your service settings:
++
+[source,text]
+----
+"rate_limit": {
+    "requests_per_minute": <<number_of_requests>>
+}
+----
+
 =====
 +
 .`service_settings` for the `openai` service
@@ -777,6 +814,28 @@ PUT _inference/text_embedding/my-msmarco-minilm-model
 <1> The `model_id` must be the ID of a text embedding model which has already been
 {ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
 
+[discrete]
+[[inference-example-mistral]]
+===== Mistral Service
+
+The following example shows how to create an {infer} endpoint called
+`mistral-embeddings-test` to perform a `text_embedding` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/mistral-embeddings-test
+{
+  "service": "mistral",
+  "service_settings": {
+    "api_key": "<api_key>",
+    "model": "mistral-embed" <1>
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The `model` must be the ID of a text embedding model which can be found in the
+https://docs.mistral.ai/getting-started/models/[Mistral models documentation]
+
 [discrete]
 [[inference-example-openai]]
 ===== OpenAI service
diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
index 89464d46744b2..e53d895882b4e 100644
--- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
+++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
@@ -1,20 +1,22 @@
 [[semantic-search-inference]]
 === Tutorial: semantic search with the {infer} API
+
 ++++
 <titleabbrev>Semantic search with the {infer} API</titleabbrev>
 ++++
 
-The instructions in this tutorial shows you how to use the {infer} API with
-various services to perform semantic search on your data. The following examples 
-use Cohere's `embed-english-v3.0` model, the `all-mpnet-base-v2` model from
-HuggingFace, and OpenAI's `text-embedding-ada-002` second generation embedding
-model. You can use any Cohere and OpenAI models, they are all supported by the
-{infer} API. For a list of supported models available on HuggingFace, refer to
+The instructions in this tutorial shows you how to use the {infer} API with various services to perform semantic search on your data.
+The following examples use Cohere's `embed-english-v3.0` model, the `all-mpnet-base-v2` model from HuggingFace, and OpenAI's `text-embedding-ada-002` second generation embedding model.
+You can use any Cohere and OpenAI models, they are all supported by the
+{infer} API.
+For a list of supported models available on HuggingFace, refer to
 <<inference-example-hugging-face-supported-models, the supported model list>>.
 
-Click the name of the service you want to use on any of the widgets below to
-review the corresponding instructions.
+Azure based examples use models available through https://ai.azure.com/explore/models?selectedTask=embeddings[Azure AI Studio]
+or https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models[Azure OpenAI].
+Mistral examples use the `mistral-embed` model from https://docs.mistral.ai/getting-started/models/[the Mistral API].
 
+Click the name of the service you want to use on any of the widgets below to review the corresponding instructions.
 
 [discrete]
 [[infer-service-requirements]]
@@ -22,7 +24,6 @@ review the corresponding instructions.
 
 include::{es-ref-dir}/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc[]
 
-
 [discrete]
 [[infer-text-embedding-task]]
 ==== Create an inference endpoint
@@ -31,49 +32,42 @@ Create an {infer} endpoint by using the <<put-inference-api>>:
 
 include::{es-ref-dir}/tab-widgets/inference-api/infer-api-task-widget.asciidoc[]
 
-
 [discrete]
 [[infer-service-mappings]]
 ==== Create the index mapping
 
-The mapping of the destination index - the index that contains the embeddings
-that the model will create based on your input text - must be created. The
-destination index must have a field with the <<dense-vector, `dense_vector`>>
+The mapping of the destination index - the index that contains the embeddings that the model will create based on your input text - must be created.
+The destination index must have a field with the <<dense-vector, `dense_vector`>>
 field type to index the output of the used model.
 
 include::{es-ref-dir}/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc[]
 
-
 [discrete]
 [[infer-service-inference-ingest-pipeline]]
 ==== Create an ingest pipeline with an inference processor
 
 Create an <<ingest,ingest pipeline>> with an
-<<inference-processor,{infer} processor>> and use the model you created above to
-infer against the data that is being ingested in the pipeline.
+<<inference-processor,{infer} processor>> and use the model you created above to infer against the data that is being ingested in the pipeline.
 
 include::{es-ref-dir}/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc[]
 
-
 [discrete]
 [[infer-load-data]]
 ==== Load data
 
-In this step, you load the data that you later use in the {infer} ingest
-pipeline to create embeddings from it.
+In this step, you load the data that you later use in the {infer} ingest pipeline to create embeddings from it.
 
-Use the `msmarco-passagetest2019-top1000` data set, which is a subset of the MS
-MARCO Passage Ranking data set. It consists of 200 queries, each accompanied by
-a list of relevant text passages. All unique passages, along with their IDs,
-have been extracted from that data set and compiled into a
+Use the `msmarco-passagetest2019-top1000` data set, which is a subset of the MS MARCO Passage Ranking data set.
+It consists of 200 queries, each accompanied by a list of relevant text passages.
+All unique passages, along with their IDs, have been extracted from that data set and compiled into a
 https://github.com/elastic/stack-docs/blob/main/docs/en/stack/ml/nlp/data/msmarco-passagetest2019-unique.tsv[tsv file].
 
 Download the file and upload it to your cluster using the
 {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer]
-in the {ml-app} UI. Assign the name `id` to the first column and `content` to
-the second column. The index name is `test-data`. Once the upload is complete,
-you can see an index named `test-data` with 182469 documents.
-
+in the {ml-app} UI.
+Assign the name `id` to the first column and `content` to the second column.
+The index name is `test-data`.
+Once the upload is complete, you can see an index named `test-data` with 182469 documents.
 
 [discrete]
 [[reindexing-data-infer]]
@@ -92,8 +86,7 @@ GET _tasks/<task_id>
 ----
 // TEST[skip:TBD]
 
-You can also cancel the reindexing process if you don't want to wait until the
-reindexing process is fully complete which might take hours for large data sets:
+You can also cancel the reindexing process if you don't want to wait until the reindexing process is fully complete which might take hours for large data sets:
 
 [source,console]
 ----
@@ -106,17 +99,14 @@ POST _tasks/<task_id>/_cancel
 [[infer-semantic-search]]
 ==== Semantic search
 
-After the data set has been enriched with the embeddings, you can query the data
-using {ref}/knn-search.html#knn-semantic-search[semantic search]. Pass a
-`query_vector_builder` to the k-nearest neighbor (kNN) vector search API, and
-provide the query text and the model you have used to create the embeddings.
+After the data set has been enriched with the embeddings, you can query the data using {ref}/knn-search.html#knn-semantic-search[semantic search].
+Pass a
+`query_vector_builder` to the k-nearest neighbor (kNN) vector search API, and provide the query text and the model you have used to create the embeddings.
 
-NOTE: If you cancelled the reindexing process, you run the query only a part of
-the data which affects the quality of your results.
+NOTE: If you cancelled the reindexing process, you run the query only a part of the data which affects the quality of your results.
 
 include::{es-ref-dir}/tab-widgets/inference-api/infer-api-search-widget.asciidoc[]
 
-
 [discrete]
 [[infer-interactive-tutorials]]
 ==== Interactive tutorials
@@ -124,4 +114,4 @@ include::{es-ref-dir}/tab-widgets/inference-api/infer-api-search-widget.asciidoc
 You can also find tutorials in an interactive Colab notebook format using the
 {es} Python client:
 * https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/inference-cohere.ipynb[Cohere {infer} tutorial notebook]
-* https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/07-inference.ipynb[OpenAI {infer} tutorial notebook]
\ No newline at end of file
+* https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/07-inference.ipynb[OpenAI {infer} tutorial notebook]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc
index 80f6da2cf602a..c8a42c4d0585a 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc
@@ -31,6 +31,12 @@
             id="infer-api-ingest-azure-ai-studio">
       Azure AI Studio
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-ingest-mistral-tab"
+            id="infer-api-ingest-mistral">
+      Mistral
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -84,6 +90,17 @@ include::infer-api-ingest-pipeline.asciidoc[tag=azure-openai]
 
 include::infer-api-ingest-pipeline.asciidoc[tag=azure-ai-studio]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-ingest-mistral-tab"
+       aria-labelledby="infer-api-ingest-mistral"
+       hidden="">
+++++
+
+include::infer-api-ingest-pipeline.asciidoc[tag=mistral]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc
index 4f85c69c7605e..a239c79e5a6d1 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc
@@ -138,3 +138,29 @@ PUT _ingest/pipeline/azure_ai_studio_embeddings
 and the `output_field` that will contain the {infer} results.
 
 // end::azure-ai-studio[]
+
+// tag::mistral[]
+
+[source,console]
+--------------------------------------------------
+PUT _ingest/pipeline/mistral_embeddings
+{
+  "processors": [
+    {
+      "inference": {
+        "model_id": "mistral_embeddings", <1>
+        "input_output": { <2>
+          "input_field": "content",
+          "output_field": "content_embedding"
+        }
+      }
+    }
+  ]
+}
+--------------------------------------------------
+<1> The name of the inference endpoint you created by using the
+<<put-inference-api>>, it's referred to as `inference_id` in that step.
+<2> Configuration object that defines the `input_field` for the {infer} process
+and the `output_field` that will contain the {infer} results.
+
+// end::mistral[]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc
index f6aa44a2b60a7..80c7c7ef23ee3 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc
@@ -31,6 +31,12 @@
             id="infer-api-mapping-azure-ai-studio">
       Azure AI Studio
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-mapping-mistral-tab"
+            id="infer-api-mapping-mistral">
+      Mistral
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -45,7 +51,8 @@ include::infer-api-mapping.asciidoc[tag=cohere]
   <div tabindex="0"
        role="tabpanel"
        id="infer-api-mapping-hf-tab"
-       aria-labelledby="infer-api-mapping-hf">
+       aria-labelledby="infer-api-mapping-hf"
+       hidden="">
 ++++
 
 include::infer-api-mapping.asciidoc[tag=hugging-face]
@@ -83,6 +90,17 @@ include::infer-api-mapping.asciidoc[tag=azure-openai]
 
 include::infer-api-mapping.asciidoc[tag=azure-ai-studio]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-mapping-mistral-tab"
+       aria-labelledby="infer-api-mapping-mistral"
+       hidden="">
+++++
+
+include::infer-api-mapping.asciidoc[tag=mistral]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc
index 8038dea713117..a1bce38a02ad2 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc
@@ -173,3 +173,37 @@ the {infer} pipeline configuration in the next step.
 <6> The field type which is text in this example.
 
 // end::azure-ai-studio[]
+
+// tag::mistral[]
+
+[source,console]
+--------------------------------------------------
+PUT mistral-embeddings
+{
+  "mappings": {
+    "properties": {
+      "content_embedding": { <1>
+        "type": "dense_vector", <2>
+        "dims": 1024, <3>
+        "element_type": "float",
+        "similarity": "dot_product" <4>
+      },
+      "content": { <5>
+        "type": "text" <6>
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> The name of the field to contain the generated tokens. It must be referenced
+in the {infer} pipeline configuration in the next step.
+<2> The field to contain the tokens is a `dense_vector` field.
+<3> The output dimensions of the model. This value may be found on the https://docs.mistral.ai/getting-started/models/[Mistral model reference].
+<4> For Mistral embeddings, the `dot_product` function should be used to
+calculate similarity.
+<5> The name of the field from which to create the dense vector representation.
+In this example, the name of the field is `content`. It must be referenced in
+the {infer} pipeline configuration in the next step.
+<6> The field type which is text in this example.
+
+// end::mistral[]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc
index a35ea4e3b0207..4face6a105819 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc
@@ -27,10 +27,16 @@
     </button>
     <button role="tab"
             aria-selected="false"
-            aria-controls="infer-api-reindex-azure-openai-tab"
-            id="infer-api-reindex-azure-openai">
+            aria-controls="infer-api-reindex-azure-ai-studio-tab"
+            id="infer-api-reindex-azure-ai-studio">
       Azure AI Studio
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-reindex-mistral-tab"
+            id="infer-api-reindex-mistral">
+      Mistral
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -84,6 +90,17 @@ include::infer-api-reindex.asciidoc[tag=azure-openai]
 
 include::infer-api-reindex.asciidoc[tag=azure-ai-studio]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-reindex-mistral-tab"
+       aria-labelledby="infer-api-reindex-mistral"
+       hidden="">
+++++
+
+include::infer-api-reindex.asciidoc[tag=mistral]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc
index a862e864fb068..927e47ea4d67c 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc
@@ -131,3 +131,26 @@ might affect the throughput of the reindexing process. If this happens, change
 `size` to `3` or a similar value in magnitude.
 
 // end::azure-ai-studio[]
+
+// tag::mistral[]
+
+[source,console]
+----
+POST _reindex?wait_for_completion=false
+{
+  "source": {
+    "index": "test-data",
+    "size": 50 <1>
+  },
+  "dest": {
+    "index": "mistral-embeddings",
+    "pipeline": "mistral_embeddings"
+  }
+}
+----
+// TEST[skip:TBD]
+<1> The default batch size for reindexing is 1000. Reducing `size` to a smaller
+number makes the update of the reindexing process quicker which enables you to
+follow the progress closely and detect errors early.
+
+// end::mistral[]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc
index 85b15678d1681..9981eb90d4929 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc
@@ -31,6 +31,12 @@
             id="infer-api-requirements-azure-ai-studio">
       Azure AI Studio
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-requirements-mistral-tab"
+            id="infer-api-requirements-mistral">
+      Mistral
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -84,6 +90,17 @@ include::infer-api-requirements.asciidoc[tag=azure-openai]
 
 include::infer-api-requirements.asciidoc[tag=azure-ai-studio]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-requirements-mistral-tab"
+       aria-labelledby="infer-api-requirements-mistral"
+       hidden="">
+++++
+
+include::infer-api-requirements.asciidoc[tag=mistral]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc
index 3ffcc6e4dd2b1..435e53bbc0bc0 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc
@@ -33,3 +33,9 @@ You can apply for access to Azure OpenAI by completing the form at https://aka.m
 * A deployed https://ai.azure.com/explore/models?selectedTask=embeddings[embeddings] or https://ai.azure.com/explore/models?selectedTask=chat-completion[chat completion] model.
 
 // end::azure-ai-studio[]
+
+// tag::mistral[]
+* A Mistral Account on https://console.mistral.ai/[La Plateforme]
+* An API key generated for your account
+
+// end::mistral[]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc
index 17b747e86be4a..6a67b28f91601 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc
@@ -31,6 +31,12 @@
             id="infer-api-search-azure-ai-studio">
       Azure AI Studio
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-search-mistral-tab"
+            id="infer-api-search-mistral">
+      Mistral
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -84,6 +90,17 @@ include::infer-api-search.asciidoc[tag=azure-openai]
 
 include::infer-api-search.asciidoc[tag=azure-ai-studio]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-search-mistral-tab"
+       aria-labelledby="infer-api-search-mistral"
+       hidden="">
+++++
+
+include::infer-api-search.asciidoc[tag=mistral]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc
index 4f1a24959de9f..523c2301e75ff 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc
@@ -340,3 +340,68 @@ query from the `azure-ai-studio-embeddings` index sorted by their proximity to t
 // NOTCONSOLE
 
 // end::azure-ai-studio[]
+
+// tag::mistral[]
+
+[source,console]
+--------------------------------------------------
+GET mistral-embeddings/_search
+{
+  "knn": {
+    "field": "content_embedding",
+    "query_vector_builder": {
+      "text_embedding": {
+        "model_id": "mistral_embeddings",
+        "model_text": "Calculate fuel cost"
+      }
+    },
+    "k": 10,
+    "num_candidates": 100
+  },
+  "_source": [
+    "id",
+    "content"
+  ]
+}
+--------------------------------------------------
+// TEST[skip:TBD]
+
+As a result, you receive the top 10 documents that are closest in meaning to the
+query from the `mistral-embeddings` index sorted by their proximity to the query:
+
+[source,consol-result]
+--------------------------------------------------
+"hits": [
+      {
+        "_index": "mistral-embeddings",
+        "_id": "DDd5OowBHxQKHyc3TDSC",
+        "_score": 0.83704096,
+        "_source": {
+          "id": 862114,
+          "body": "How to calculate fuel cost for a road trip. By Tara Baukus Mello • Bankrate.com. Dear Driving for Dollars, My family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost.It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes.y family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost. It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes."
+        }
+      },
+      {
+        "_index": "mistral-embeddings",
+        "_id": "ajd5OowBHxQKHyc3TDSC",
+        "_score": 0.8345704,
+        "_source": {
+          "id": 820622,
+          "body": "Home Heating Calculator. Typically, approximately 50% of the energy consumed in a home annually is for space heating. When deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important.This calculator can help you estimate the cost of fuel for different heating appliances.hen deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important. This calculator can help you estimate the cost of fuel for different heating appliances."
+        }
+      },
+      {
+        "_index": "mistral-embeddings",
+        "_id": "Djd5OowBHxQKHyc3TDSC",
+        "_score": 0.8327426,
+        "_source": {
+          "id": 8202683,
+          "body": "Fuel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel.If you are paying $4 per gallon, the trip would cost you $200.Most boats have much larger gas tanks than cars.uel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel."
+        }
+      },
+      (...)
+    ]
+--------------------------------------------------
+// NOTCONSOLE
+
+// end::mistral[]
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc
index 3bccb140d44f6..1f3ad645d7c29 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc
@@ -31,6 +31,12 @@
             id="infer-api-task-azure-ai-studio">
       Azure AI Studio
     </button>
+    <button role="tab"
+            aria-selected="false"
+            aria-controls="infer-api-task-mistral-tab"
+            id="infer-api-task-mistral">
+      Mistral
+    </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
@@ -84,6 +90,17 @@ include::infer-api-task.asciidoc[tag=azure-openai]
 
 include::infer-api-task.asciidoc[tag=azure-ai-studio]
 
+++++
+  </div>
+  <div tabindex="0"
+       role="tabpanel"
+       id="infer-api-task-mistral-tab"
+       aria-labelledby="infer-api-task-mistral"
+       hidden="">
+++++
+
+include::infer-api-task.asciidoc[tag=mistral]
+
 ++++
   </div>
 </div>
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc
index 5692388a18531..18fa3ba541bff 100644
--- a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc
+++ b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc
@@ -157,3 +157,23 @@ Also, when using this model the recommended similarity measure to use in the
 `dense_vector` field mapping is `dot_product`.
 
 // end::azure-ai-studio[]
+
+// tag::mistral[]
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/mistral_embeddings <1>
+{
+    "service": "mistral",
+    "service_settings": {
+        "api_key": "<api_key>", <2>
+        "model": "<model_id>" <3>
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {infer} endpoint is `mistral_embeddings`.
+<2> The API key for accessing the Mistral API. You can find this in your Mistral account's API Keys page.
+<3> The Mistral embeddings model name, for example `mistral-embed`.
+
+// end::mistral[]

From ac6c0eecc1f2cb75a6d040306b6373d666f64577 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Wed, 5 Jun 2024 17:32:14 +0200
Subject: [PATCH 175/208] Ensure synthetic source and dv codec are enabled with
 logs index mode (attempt 2). (#109382)

This was initially muted via #109365, because of a failing newly introduced assert.

Original PR #109269
---
 .../test/aggregations/time_series.yml         |  6 +--
 .../rest-api-spec/test/logsdb/10_settings.yml |  4 ++
 .../cluster/metadata/IndexMetadata.java       |  7 +--
 .../index/mapper/SourceFieldMapper.java       | 43 +++++++++++++----
 .../index/LogsIndexModeTests.java             | 31 +++++++-----
 .../index/codec/PerFieldMapperCodecTests.java | 47 +++++++++++++++----
 .../index/mapper/SourceFieldMapperTests.java  |  7 +++
 .../index/mapper/MapperServiceTestCase.java   |  5 ++
 8 files changed, 116 insertions(+), 34 deletions(-)

diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
index 7800923ab1580..1703d4908a753 100644
--- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
+++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
@@ -294,11 +294,11 @@ setup:
 ---
 "Configure with no synthetic source":
   - requires:
-      cluster_features: ["gte_v8.9.0"]
-      reason: "Error message fix in 8.9"
+      cluster_features: ["gte_v8.15.0"]
+      reason: "Error message changed in 8.15.0"
 
   - do:
-      catch: '/Time series indices only support synthetic source./'
+      catch: '/Indices with with index mode \[time_series\] only support synthetic source/'
       indices.create:
         index: tsdb_error
         body:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
index 95075da20fe5e..128903f4faac8 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
@@ -77,6 +77,10 @@ create logs index:
   - is_true: test
   - match: { test.settings.index.mode: "logs" }
 
+  - do:
+      indices.get_mapping:
+        index: test
+  - match: { test.mappings._source.mode: synthetic }
 
 ---
 using default timestamp field mapping:
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
index 681dcb3e314e3..64809c963cb6d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -2267,8 +2267,9 @@ IndexMetadata build(boolean repair) {
             }
 
             final boolean isSearchableSnapshot = SearchableSnapshotsSettings.isSearchableSnapshotStore(settings);
-            final String indexMode = settings.get(IndexSettings.MODE.getKey());
-            final boolean isTsdb = indexMode != null && IndexMode.TIME_SERIES.getName().equals(indexMode.toLowerCase(Locale.ROOT));
+            String indexModeString = settings.get(IndexSettings.MODE.getKey());
+            final IndexMode indexMode = indexModeString != null ? IndexMode.fromString(indexModeString.toLowerCase(Locale.ROOT)) : null;
+            final boolean isTsdb = indexMode == IndexMode.TIME_SERIES;
             return new IndexMetadata(
                 new Index(index, uuid),
                 version,
@@ -2308,7 +2309,7 @@ IndexMetadata build(boolean repair) {
                 AutoExpandReplicas.SETTING.get(settings),
                 isSearchableSnapshot,
                 isSearchableSnapshot && settings.getAsBoolean(SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY, false),
-                isTsdb ? IndexMode.TIME_SERIES : null,
+                indexMode,
                 isTsdb ? IndexSettings.TIME_SERIES_START_TIME.get(settings) : null,
                 isTsdb ? IndexSettings.TIME_SERIES_END_TIME.get(settings) : null,
                 SETTING_INDEX_VERSION_COMPATIBILITY.get(settings),
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index b819ffb0ef6ad..d6a15ff9ec47a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -69,6 +69,14 @@ private enum Mode {
         IndexMode.TIME_SERIES
     );
 
+    private static final SourceFieldMapper LOGS_DEFAULT = new SourceFieldMapper(
+        Mode.SYNTHETIC,
+        Explicit.IMPLICIT_TRUE,
+        Strings.EMPTY_ARRAY,
+        Strings.EMPTY_ARRAY,
+        IndexMode.LOGS
+    );
+
     /*
      * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below
      * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version.
@@ -156,7 +164,8 @@ protected Parameter<?>[] getParameters() {
 
         private boolean isDefault() {
             Mode m = mode.get();
-            if (m != null && (((indexMode == IndexMode.TIME_SERIES && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
+            if (m != null
+                && (((indexMode != null && indexMode.isSyntheticSourceEnabled() && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
                 return false;
             }
             return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty();
@@ -165,15 +174,19 @@ private boolean isDefault() {
         @Override
         public SourceFieldMapper build() {
             if (enabled.getValue().explicit()) {
-                if (indexMode == IndexMode.TIME_SERIES) {
-                    throw new MapperParsingException("Time series indices only support synthetic source");
+                if (indexMode != null && indexMode.isSyntheticSourceEnabled()) {
+                    throw new MapperParsingException("Indices with with index mode [" + indexMode + "] only support synthetic source");
                 }
                 if (mode.get() != null) {
                     throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
                 }
             }
             if (isDefault()) {
-                return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT;
+                return switch (indexMode) {
+                    case TIME_SERIES -> TSDB_DEFAULT;
+                    case LOGS -> LOGS_DEFAULT;
+                    default -> DEFAULT;
+                };
             }
             if (supportsNonDefaultParameterValues == false) {
                 List<String> disallowed = new ArrayList<>();
@@ -212,10 +225,21 @@ public SourceFieldMapper build() {
 
     }
 
-    public static final TypeParser PARSER = new ConfigurableTypeParser(
-        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES
-            ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT
-            : DEFAULT,
+    public static final TypeParser PARSER = new ConfigurableTypeParser(c -> {
+        var indexMode = c.getIndexSettings().getMode();
+        if (indexMode.isSyntheticSourceEnabled()) {
+            if (indexMode == IndexMode.TIME_SERIES) {
+                if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) {
+                    return TSDB_DEFAULT;
+                } else {
+                    return TSDB_LEGACY_DEFAULT;
+                }
+            } else if (indexMode == IndexMode.LOGS) {
+                return LOGS_DEFAULT;
+            }
+        }
+        return DEFAULT;
+    },
         c -> new Builder(
             c.getIndexSettings().getMode(),
             c.getSettings(),
@@ -323,6 +347,9 @@ public void preParse(DocumentParserContext context) throws IOException {
         final BytesReference adaptedSource = applyFilters(originalSource, contentType);
 
         if (adaptedSource != null) {
+            assert context.indexSettings().getIndexVersionCreated().before(IndexVersions.V_8_7_0)
+                || indexMode == null
+                || indexMode.isSyntheticSourceEnabled() == false;
             final BytesRef ref = adaptedSource.toBytesRef();
             context.doc().add(new StoredField(fieldType().name(), ref.bytes, ref.offset, ref.length));
         }
diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
index fd73a8c9f8f52..caddc7d5ea5af 100644
--- a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
@@ -10,12 +10,13 @@
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.MapperServiceTestCase;
-import org.hamcrest.Matchers;
+import org.elasticsearch.test.ESTestCase;
 
-public class LogsIndexModeTests extends MapperServiceTestCase {
+import static org.hamcrest.Matchers.equalTo;
+
+public class LogsIndexModeTests extends ESTestCase {
     public void testLogsIndexModeSetting() {
-        assertThat(IndexSettings.MODE.get(buildSettings()), Matchers.equalTo(IndexMode.LOGS));
+        assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGS));
     }
 
     public void testSortField() {
@@ -24,8 +25,10 @@ public void testSortField() {
             .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
     }
 
     public void testSortMode() {
@@ -35,9 +38,11 @@ public void testSortMode() {
             .put(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "max")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("max", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("max", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey())));
     }
 
     public void testSortOrder() {
@@ -47,9 +52,11 @@ public void testSortOrder() {
             .put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("desc", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("desc", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey())));
     }
 
     public void testSortMissing() {
@@ -59,9 +66,11 @@ public void testSortMissing() {
             .put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last")
             .build();
         final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", sortSettings);
+        assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGS));
         final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
-        assertThat("agent_id", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
-        assertThat("_last", Matchers.equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey())));
+        assertThat(settings.getMode(), equalTo(IndexMode.LOGS));
+        assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
+        assertThat("_last", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey())));
     }
 
     private Settings buildSettings() {
diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
index 74657842488b5..525fa31673494 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat;
@@ -61,6 +62,28 @@ public class PerFieldMapperCodecTests extends ESTestCase {
         }
         """;
 
+    private static final String MAPPING_3 = """
+        {
+            "_data_stream_timestamp": {
+                "enabled": true
+            },
+            "properties": {
+                "@timestamp": {
+                    "type": "date"
+                },
+                "hostname": {
+                    "type": "keyword"
+                },
+                "response_size": {
+                    "type": "long"
+                },
+                "message": {
+                    "type": "text"
+                }
+            }
+        }
+        """;
+
     public void testUseBloomFilter() throws IOException {
         PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, randomBoolean(), false);
         assertThat(perFieldMapperCodec.useBloomFilter("_id"), is(true));
@@ -103,13 +126,13 @@ public void testDoNotUseES87TSDBEncodingForTimestampFieldNonTimeSeriesIndex() th
     }
 
     public void testEnableES87TSDBCodec() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, true, MAPPING_1);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.TIME_SERIES, MAPPING_1);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
     }
 
     public void testDisableES87TSDBCodec() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, true, MAPPING_1);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, IndexMode.TIME_SERIES, MAPPING_1);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
     }
@@ -144,31 +167,37 @@ private PerFieldFormatSupplier createFormatSupplier(boolean timestampField, bool
     }
 
     public void testUseES87TSDBEncodingSettingDisabled() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, true, MAPPING_2);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(false, IndexMode.TIME_SERIES, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
     }
 
     public void testUseTimeSeriesModeDisabledCodecDisabled() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, false, MAPPING_2);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.STANDARD, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
     }
 
     public void testUseTimeSeriesModeAndCodecEnabled() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, true, MAPPING_2);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.TIME_SERIES, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
     }
 
-    private PerFieldFormatSupplier createFormatSupplier(boolean enableES87TSDBCodec, boolean timeSeries, String mapping)
-        throws IOException {
+    public void testLogsIndexMode() throws IOException {
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGS, MAPPING_3);
+        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
+        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("hostname")), is(true));
+        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("response_size")), is(true));
+    }
+
+    private PerFieldFormatSupplier createFormatSupplier(boolean enableES87TSDBCodec, IndexMode mode, String mapping) throws IOException {
         Settings.Builder settings = Settings.builder();
-        if (timeSeries) {
-            settings.put(IndexSettings.MODE.getKey(), "time_series");
+        settings.put(IndexSettings.MODE.getKey(), mode);
+        if (mode == IndexMode.TIME_SERIES) {
             settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field");
         }
         settings.put(IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING.getKey(), enableES87TSDBCodec);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
index 802a18645eab6..d0350c1d92a83 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
@@ -243,6 +243,13 @@ public void testSyntheticSourceInTimeSeries() throws IOException {
         assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString());
     }
 
+    public void testSyntheticSourceWithLogsIndexMode() throws IOException {
+        XContentBuilder mapping = fieldMapping(b -> { b.field("type", "keyword"); });
+        DocumentMapper mapper = createLogsModeDocumentMapper(mapping);
+        assertTrue(mapper.sourceMapper().isSynthetic());
+        assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString());
+    }
+
     public void testSupportsNonDefaultParameterValues() throws IOException {
         Settings settings = Settings.builder().put(SourceFieldMapper.LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, false).build();
         {
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
index 388d8d6fa6ffd..dfd4a59e2c3a1 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
@@ -148,6 +148,11 @@ protected final DocumentMapper createTimeSeriesModeDocumentMapper(XContentBuilde
         return createMapperService(settings, mappings).documentMapper();
     }
 
+    protected final DocumentMapper createLogsModeDocumentMapper(XContentBuilder mappings) throws IOException {
+        Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), "logs").build();
+        return createMapperService(settings, mappings).documentMapper();
+    }
+
     protected final DocumentMapper createDocumentMapper(IndexVersion version, XContentBuilder mappings) throws IOException {
         return createMapperService(version, mappings).documentMapper();
     }

From 17c6230b9f314c6abeba88162981cfd458abbd09 Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Thu, 6 Jun 2024 01:32:17 +1000
Subject: [PATCH 176/208] Remove 8.13 from branches.json

---
 branches.json | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/branches.json b/branches.json
index daf6d249f7268..2794b545facc6 100644
--- a/branches.json
+++ b/branches.json
@@ -7,9 +7,6 @@
     {
       "branch": "8.14"
     },
-    {
-      "branch": "8.13"
-    },
     {
       "branch": "7.17"
     }

From 307cac564802960a61a5626f1a40f59301d82e46 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 5 Jun 2024 14:08:44 -0400
Subject: [PATCH 177/208] ESQL: Move `NamedExpression` serialization (#109380)

This moves the serialization for the remaining `NamedExpression`
subclass into the class itself, and switches all direct serialization of
`NamedExpression`s to `readNamedWriteable` and friends. All other
`NamedExpression` subclasses extend from `Attribute` who's serialization
was moved ealier. They are already registered under the "category class"
for `Attribute`. This also registers them as `NamedExpression`s.
---
 .../xpack/esql/core/expression/Alias.java     |  50 ++++++-
 .../xpack/esql/core/expression/Attribute.java |   3 +-
 .../esql/core/expression/NamedExpression.java |  13 +-
 .../esql/core/expression/UnresolvedAlias.java |  78 -----------
 .../esql/core/expression/UnresolvedStar.java  |  12 ++
 .../xpack/esql/core/util/PlanStreamInput.java |   8 ++
 .../esql/core/util/PlanStreamOutput.java      |  27 ++++
 .../esql/enrich/EnrichLookupService.java      |   7 +-
 .../esql/expression/NamedExpressions.java     |   1 -
 .../expression/UnresolvedNamePattern.java     |  12 ++
 .../function/UnsupportedAttribute.java        |   6 +
 .../xpack/esql/io/stream/PlanNamedTypes.java  | 122 +++++++-----------
 .../xpack/esql/io/stream/PlanStreamInput.java |   6 +-
 .../esql/io/stream/PlanStreamOutput.java      |   8 +-
 .../xpack/esql/plugin/EsqlPlugin.java         |   3 +
 .../xpack/esql/SerializationTestUtils.java    |   3 +
 .../xpack/esql/expression/AliasTests.java     |  86 ++++++++++++
 .../function/ReferenceAttributeTests.java     |   8 +-
 .../esql/io/stream/PlanNamedTypesTests.java   |  11 --
 19 files changed, 276 insertions(+), 188 deletions(-)
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAlias.java
 create mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java
 create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java
index 58203c8a0072e..d9f99b6d92318 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java
@@ -6,11 +6,18 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
+import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
 
+import java.io.IOException;
 import java.util.List;
+import java.util.Objects;
 
 import static java.util.Collections.singletonList;
 
@@ -22,7 +29,8 @@
  * And in {@code SELECT col AS x} "col" is a named expression that gets renamed to "x" through an alias.
  *
  */
-public class Alias extends NamedExpression {
+public final class Alias extends NamedExpression {
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(NamedExpression.class, "Alias", Alias::new);
 
     private final Expression child;
     private final String qualifier;
@@ -51,6 +59,32 @@ public Alias(Source source, String name, String qualifier, Expression child, Nam
         this.qualifier = qualifier;
     }
 
+    public Alias(StreamInput in) throws IOException {
+        this(
+            Source.readFrom((StreamInput & PlanStreamInput) in),
+            in.readString(),
+            in.readOptionalString(),
+            ((PlanStreamInput) in).readExpression(),
+            NameId.readFrom((StreamInput & PlanStreamInput) in),
+            in.readBoolean()
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Source.EMPTY.writeTo(out);
+        out.writeString(name());
+        out.writeOptionalString(qualifier());
+        ((PlanStreamOutput) out).writeExpression(child());
+        id().writeTo(out);
+        out.writeBoolean(synthetic());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
     @Override
     protected NodeInfo<Alias> info() {
         return NodeInfo.create(this, Alias::new, name(), qualifier, child, id(), synthetic());
@@ -113,4 +147,18 @@ public String nodeString() {
     public static Expression unwrap(Expression e) {
         return e instanceof Alias as ? as.child() : e;
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (super.equals(obj) == false) {
+            return false;
+        }
+        Alias other = (Alias) obj;
+        return Objects.equals(qualifier, other.qualifier);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), qualifier);
+    }
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
index 5326825ec1105..e89f39294a28b 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java
@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
-import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -30,7 +29,7 @@
  * is a named expression (an {@code Alias} will be created automatically for it).
  * The rest are not as they are not part of the projection and thus are not part of the derived table.
  */
-public abstract class Attribute extends NamedExpression implements NamedWriteable {
+public abstract class Attribute extends NamedExpression {
     public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         // TODO add UnsupportedAttribute when these are moved to the same project
         return List.of(FieldAttribute.ENTRY, MetadataAttribute.ENTRY, ReferenceAttribute.ENTRY);
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java
index 4a3666c8b8aa7..e3e9a60180da7 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java
@@ -6,8 +6,11 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
@@ -15,7 +18,15 @@
  * An expression that has a name. Named expressions can be used as a result
  * (by converting to an attribute).
  */
-public abstract class NamedExpression extends Expression {
+public abstract class NamedExpression extends Expression implements NamedWriteable {
+    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        for (NamedWriteableRegistry.Entry e : Attribute.getNamedWriteables()) {
+            entries.add(new NamedWriteableRegistry.Entry(NamedExpression.class, e.name, in -> (NamedExpression) e.reader.read(in)));
+        }
+        entries.add(Alias.ENTRY);
+        return entries;
+    }
 
     private final String name;
     private final NameId id;
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAlias.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAlias.java
deleted file mode 100644
index a4b0d06f54b83..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAlias.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.expression;
-
-import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
-import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-
-import java.util.List;
-import java.util.Objects;
-
-import static java.util.Collections.singletonList;
-
-public class UnresolvedAlias extends UnresolvedNamedExpression {
-
-    private final Expression child;
-
-    public UnresolvedAlias(Source source, Expression child) {
-        super(source, singletonList(child));
-        this.child = child;
-    }
-
-    @Override
-    protected NodeInfo<UnresolvedAlias> info() {
-        return NodeInfo.create(this, UnresolvedAlias::new, child);
-    }
-
-    @Override
-    public Expression replaceChildren(List<Expression> newChildren) {
-        return new UnresolvedAlias(source(), newChildren.get(0));
-    }
-
-    public Expression child() {
-        return child;
-    }
-
-    @Override
-    public String unresolvedMessage() {
-        return "Unknown alias [" + name() + "]";
-    }
-
-    @Override
-    public Nullability nullable() {
-        throw new UnresolvedException("nullable", this);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(child);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        /*
-         * Intentionally not calling the superclass
-         * equals because it uses id which we always
-         * mutate when we make a clone.
-         */
-        if (obj == null || obj.getClass() != getClass()) {
-            return false;
-        }
-        return Objects.equals(child, ((UnresolvedAlias) obj).child);
-    }
-
-    @Override
-    public String toString() {
-        return child + " AS ?";
-    }
-
-    @Override
-    public String nodeString() {
-        return toString();
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java
index 198016c710ce3..f3b52cfcccf90 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java
@@ -6,10 +6,12 @@
  */
 package org.elasticsearch.xpack.esql.core.expression;
 
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
@@ -25,6 +27,16 @@ public UnresolvedStar(Source source, UnresolvedAttribute qualifier) {
         this.qualifier = qualifier;
     }
 
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        throw new UnsupportedOperationException("doesn't escape the node");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("doesn't escape the node");
+    }
+
     @Override
     protected NodeInfo<UnresolvedStar> info() {
         return NodeInfo.create(this, UnresolvedStar::new, qualifier);
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
index df8fac06dd478..485084bac60b3 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.esql.core.util;
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 
@@ -33,4 +35,10 @@ public interface PlanStreamInput {
      * the same result.
      */
     NameId mapNameId(long id) throws IOException;
+
+    /**
+     * Read an {@link Expression} from the stream. This will soon be replaced with
+     * {@link StreamInput#readNamedWriteable}.
+     */
+    Expression readExpression() throws IOException;
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java
new file mode 100644
index 0000000000000..6a3d8fb77316c
--- /dev/null
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamOutput.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.util;
+
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+
+import java.io.IOException;
+
+/**
+ * Interface for streams that can serialize plan components. This exists so
+ * ESQL proper can expose streaming capability to ESQL-core. If the world is kind
+ * and just we'll remove this when we flatten everything from ESQL-core into
+ * ESQL proper.
+ */
+public interface PlanStreamOutput {
+    /**
+     * Write an {@link Expression} to the stream. This will soon be replaced with
+     * {@link StreamOutput#writeNamedWriteable}.
+     */
+    void writeExpression(Expression expression) throws IOException;
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
index 69d988c958169..05b78c8b5f309 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java
@@ -96,9 +96,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
-import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader;
-import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter;
-
 /**
  * {@link EnrichLookupService} performs enrich lookup for a given input page. The lookup process consists of three stages:
  * - Stage 1: Finding matching document IDs for the input page. This stage is done by the {@link EnrichQuerySourceOperator} or its variants.
@@ -460,7 +457,7 @@ private static class LookupRequest extends TransportRequest implements IndicesRe
             }
             this.toRelease = inputPage;
             PlanStreamInput planIn = new PlanStreamInput(in, PlanNameRegistry.INSTANCE, in.namedWriteableRegistry(), null);
-            this.extractFields = planIn.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readNamedExpression));
+            this.extractFields = planIn.readNamedWriteableCollectionAsList(NamedExpression.class);
         }
 
         @Override
@@ -475,7 +472,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(matchField);
             out.writeWriteable(inputPage);
             PlanStreamOutput planOut = new PlanStreamOutput(out, PlanNameRegistry.INSTANCE, null);
-            planOut.writeCollection(extractFields, writerFromPlanWriter(PlanStreamOutput::writeNamedExpression));
+            planOut.writeNamedWriteableCollection(extractFields);
         }
 
         @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java
index cb6aaf879f3cb..d0c8adfd3c858 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/NamedExpressions.java
@@ -17,7 +17,6 @@
 import java.util.Map;
 
 public class NamedExpressions {
-
     /**
      * Calculates the actual output of a command given the new attributes plus the existing inputs that are emitted as outputs
      * @param fields the fields added by the command
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/UnresolvedNamePattern.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/UnresolvedNamePattern.java
index 7df28f0648318..98282b5dec0eb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/UnresolvedNamePattern.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/UnresolvedNamePattern.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.esql.expression;
 
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
@@ -16,6 +17,7 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
@@ -42,6 +44,16 @@ public UnresolvedNamePattern(Source source, CharacterRunAutomaton automaton, Str
         this.name = name;
     }
 
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        throw new UnsupportedOperationException("doesn't escape the node");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("doesn't escape the node");
+    }
+
     public boolean match(String string) {
         return automaton.run(string);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
index fe6db916f7a0d..79dcc6a3d3920 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -36,6 +37,11 @@ public final class UnsupportedAttribute extends FieldAttribute implements Unreso
         "UnsupportedAttribute",
         UnsupportedAttribute::new
     );
+    public static final NamedWriteableRegistry.Entry NAMED_EXPRESSION_ENTRY = new NamedWriteableRegistry.Entry(
+        NamedExpression.class,
+        ENTRY.name,
+        UnsupportedAttribute::new
+    );
 
     private final String message;
     private final boolean hasCustomMessage; // TODO remove me and just use message != null?
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index fe2c704743f5c..624c9f5c65ca5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
-import org.elasticsearch.xpack.esql.core.expression.NameId;
 import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
@@ -299,12 +298,12 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(LogicalPlan.class, Project.class, PlanNamedTypes::writeProject, PlanNamedTypes::readProject),
             of(LogicalPlan.class, TopN.class, PlanNamedTypes::writeTopN, PlanNamedTypes::readTopN),
             // Attributes
-            of(NamedExpression.class, FieldAttribute.class, (o, a) -> a.writeTo(o), FieldAttribute::new),
-            of(NamedExpression.class, ReferenceAttribute.class, (o, a) -> a.writeTo(o), ReferenceAttribute::new),
-            of(NamedExpression.class, MetadataAttribute.class, (o, a) -> a.writeTo(o), MetadataAttribute::new),
-            of(NamedExpression.class, UnsupportedAttribute.class, (o, a) -> a.writeTo(o), UnsupportedAttribute::new),
+            of(Expression.class, FieldAttribute.class, (o, a) -> a.writeTo(o), FieldAttribute::new),
+            of(Expression.class, ReferenceAttribute.class, (o, a) -> a.writeTo(o), ReferenceAttribute::new),
+            of(Expression.class, MetadataAttribute.class, (o, a) -> a.writeTo(o), MetadataAttribute::new),
+            of(Expression.class, UnsupportedAttribute.class, (o, a) -> a.writeTo(o), UnsupportedAttribute::new),
             // NamedExpressions
-            of(NamedExpression.class, Alias.class, PlanNamedTypes::writeAlias, PlanNamedTypes::readAlias),
+            of(Expression.class, Alias.class, (o, a) -> a.writeTo(o), Alias::new),
             // BinaryComparison
             of(EsqlBinaryComparison.class, Equals.class, PlanNamedTypes::writeBinComparison, PlanNamedTypes::readBinComparison),
             of(EsqlBinaryComparison.class, NotEquals.class, PlanNamedTypes::writeBinComparison, PlanNamedTypes::readBinComparison),
@@ -454,7 +453,7 @@ static AggregateExec readAggregateExec(PlanStreamInput in) throws IOException {
             Source.readFrom(in),
             in.readPhysicalPlanNode(),
             in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)),
-            readNamedExpressions(in),
+            in.readNamedWriteableCollectionAsList(NamedExpression.class),
             in.readEnum(AggregateExec.Mode.class),
             in.readOptionalVInt()
         );
@@ -464,7 +463,7 @@ static void writeAggregateExec(PlanStreamOutput out, AggregateExec aggregateExec
         Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(aggregateExec.child());
         out.writeCollection(aggregateExec.groupings(), writerFromPlanWriter(PlanStreamOutput::writeExpression));
-        writeNamedExpressions(out, aggregateExec.aggregates());
+        out.writeNamedWriteableCollection(aggregateExec.aggregates());
         out.writeEnum(aggregateExec.getMode());
         out.writeOptionalVInt(aggregateExec.estimatedRowSize());
     }
@@ -547,19 +546,19 @@ static void writeIndexMode(StreamOutput out, IndexMode indexMode) throws IOExcep
     }
 
     static EvalExec readEvalExec(PlanStreamInput in) throws IOException {
-        return new EvalExec(Source.readFrom(in), in.readPhysicalPlanNode(), readAliases(in));
+        return new EvalExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readCollectionAsList(Alias::new));
     }
 
     static void writeEvalExec(PlanStreamOutput out, EvalExec evalExec) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(evalExec.child());
-        writeAliases(out, evalExec.fields());
+        out.writeCollection(evalExec.fields());
     }
 
     static EnrichExec readEnrichExec(PlanStreamInput in) throws IOException {
         final Source source = Source.readFrom(in);
         final PhysicalPlan child = in.readPhysicalPlanNode();
-        final NamedExpression matchField = in.readNamedExpression();
+        final NamedExpression matchField = in.readNamedWriteable(NamedExpression.class);
         final String policyName = in.readString();
         final String matchType = (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_EXTENDED_ENRICH_TYPES))
             ? in.readString()
@@ -587,14 +586,14 @@ static EnrichExec readEnrichExec(PlanStreamInput in) throws IOException {
             policyName,
             policyMatchField,
             concreteIndices,
-            readNamedExpressions(in)
+            in.readNamedWriteableCollectionAsList(NamedExpression.class)
         );
     }
 
     static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(enrich.child());
-        out.writeNamedExpression(enrich.matchField());
+        out.writeNamedWriteable(enrich.matchField());
         out.writeString(enrich.policyName());
         if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_EXTENDED_ENRICH_TYPES)) {
             out.writeString(enrich.matchType());
@@ -611,7 +610,7 @@ static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOEx
                 throw new IllegalStateException("expected a single concrete enrich index; got " + enrich.concreteIndices());
             }
         }
-        writeNamedExpressions(out, enrich.enrichFields());
+        out.writeNamedWriteableCollection(enrich.enrichFields());
     }
 
     static ExchangeExec readExchangeExec(PlanStreamInput in) throws IOException {
@@ -728,7 +727,7 @@ static MvExpandExec readMvExpandExec(PlanStreamInput in) throws IOException {
         return new MvExpandExec(
             Source.readFrom(in),
             in.readPhysicalPlanNode(),
-            in.readNamedExpression(),
+            in.readNamedWriteable(NamedExpression.class),
             in.readNamedWriteable(Attribute.class)
         );
     }
@@ -736,7 +735,7 @@ static MvExpandExec readMvExpandExec(PlanStreamInput in) throws IOException {
     static void writeMvExpandExec(PlanStreamOutput out, MvExpandExec mvExpandExec) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(mvExpandExec.child());
-        out.writeNamedExpression(mvExpandExec.target());
+        out.writeNamedWriteable(mvExpandExec.target());
         out.writeNamedWriteable(mvExpandExec.expanded());
     }
 
@@ -755,23 +754,27 @@ static void writeOrderExec(PlanStreamOutput out, OrderExec orderExec) throws IOE
     }
 
     static ProjectExec readProjectExec(PlanStreamInput in) throws IOException {
-        return new ProjectExec(Source.readFrom(in), in.readPhysicalPlanNode(), readNamedExpressions(in));
+        return new ProjectExec(
+            Source.readFrom(in),
+            in.readPhysicalPlanNode(),
+            in.readNamedWriteableCollectionAsList(NamedExpression.class)
+        );
     }
 
     static void writeProjectExec(PlanStreamOutput out, ProjectExec projectExec) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writePhysicalPlanNode(projectExec.child());
-        writeNamedExpressions(out, projectExec.projections());
+        out.writeNamedWriteableCollection(projectExec.projections());
     }
 
     static RowExec readRowExec(PlanStreamInput in) throws IOException {
-        return new RowExec(Source.readFrom(in), readAliases(in));
+        return new RowExec(Source.readFrom(in), in.readCollectionAsList(Alias::new));
     }
 
     static void writeRowExec(PlanStreamOutput out, RowExec rowExec) throws IOException {
         assert rowExec.children().size() == 0;
         Source.EMPTY.writeTo(out);
-        writeAliases(out, rowExec.fields());
+        out.writeCollection(rowExec.fields());
     }
 
     @SuppressWarnings("unchecked")
@@ -813,7 +816,7 @@ static Aggregate readAggregate(PlanStreamInput in) throws IOException {
             Source.readFrom(in),
             in.readLogicalPlanNode(),
             in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)),
-            readNamedExpressions(in)
+            in.readNamedWriteableCollectionAsList(NamedExpression.class)
         );
     }
 
@@ -821,7 +824,7 @@ static void writeAggregate(PlanStreamOutput out, Aggregate aggregate) throws IOE
         Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(aggregate.child());
         out.writeCollection(aggregate.groupings(), writerFromPlanWriter(PlanStreamOutput::writeExpression));
-        writeNamedExpressions(out, aggregate.aggregates());
+        out.writeNamedWriteableCollection(aggregate.aggregates());
     }
 
     static Dissect readDissect(PlanStreamInput in) throws IOException {
@@ -890,13 +893,13 @@ private static void writeEsSourceOptions(PlanStreamOutput out) throws IOExceptio
     }
 
     static Eval readEval(PlanStreamInput in) throws IOException {
-        return new Eval(Source.readFrom(in), in.readLogicalPlanNode(), readAliases(in));
+        return new Eval(Source.readFrom(in), in.readLogicalPlanNode(), in.readCollectionAsList(Alias::new));
     }
 
     static void writeEval(PlanStreamOutput out, Eval eval) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(eval.child());
-        writeAliases(out, eval.fields());
+        out.writeCollection(eval.fields());
     }
 
     static Enrich readEnrich(PlanStreamInput in) throws IOException {
@@ -907,7 +910,7 @@ static Enrich readEnrich(PlanStreamInput in) throws IOException {
         final Source source = Source.readFrom(in);
         final LogicalPlan child = in.readLogicalPlanNode();
         final Expression policyName = in.readExpression();
-        final NamedExpression matchField = in.readNamedExpression();
+        final NamedExpression matchField = in.readNamedWriteable(NamedExpression.class);
         if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) {
             in.readString(); // discard the old policy name
         }
@@ -922,7 +925,16 @@ static Enrich readEnrich(PlanStreamInput in) throws IOException {
             }
             concreteIndices = Map.of(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, Iterables.get(esIndex.concreteIndices(), 0));
         }
-        return new Enrich(source, child, mode, policyName, matchField, policy, concreteIndices, readNamedExpressions(in));
+        return new Enrich(
+            source,
+            child,
+            mode,
+            policyName,
+            matchField,
+            policy,
+            concreteIndices,
+            in.readNamedWriteableCollectionAsList(NamedExpression.class)
+        );
     }
 
     static void writeEnrich(PlanStreamOutput out, Enrich enrich) throws IOException {
@@ -933,7 +945,7 @@ static void writeEnrich(PlanStreamOutput out, Enrich enrich) throws IOException
         Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(enrich.child());
         out.writeExpression(enrich.policyName());
-        out.writeNamedExpression(enrich.matchField());
+        out.writeNamedWriteable(enrich.matchField());
         if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) {
             out.writeString(BytesRefs.toString(enrich.policyName().fold())); // old policy name
         }
@@ -950,17 +962,17 @@ static void writeEnrich(PlanStreamOutput out, Enrich enrich) throws IOException
                 throw new IllegalStateException("expected a single enrich index; got " + concreteIndices);
             }
         }
-        writeNamedExpressions(out, enrich.enrichFields());
+        out.writeNamedWriteableCollection(enrich.enrichFields());
     }
 
     static EsqlProject readEsqlProject(PlanStreamInput in) throws IOException {
-        return new EsqlProject(Source.readFrom(in), in.readLogicalPlanNode(), readNamedExpressions(in));
+        return new EsqlProject(Source.readFrom(in), in.readLogicalPlanNode(), in.readNamedWriteableCollectionAsList(NamedExpression.class));
     }
 
     static void writeEsqlProject(PlanStreamOutput out, EsqlProject project) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(project.child());
-        writeNamedExpressions(out, project.projections());
+        out.writeNamedWriteableCollection(project.projections());
     }
 
     static Filter readFilter(PlanStreamInput in) throws IOException {
@@ -1006,7 +1018,7 @@ static MvExpand readMvExpand(PlanStreamInput in) throws IOException {
         return new MvExpand(
             Source.readFrom(in),
             in.readLogicalPlanNode(),
-            in.readNamedExpression(),
+            in.readNamedWriteable(NamedExpression.class),
             in.readNamedWriteable(Attribute.class)
         );
     }
@@ -1014,7 +1026,7 @@ static MvExpand readMvExpand(PlanStreamInput in) throws IOException {
     static void writeMvExpand(PlanStreamOutput out, MvExpand mvExpand) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(mvExpand.child());
-        out.writeNamedExpression(mvExpand.target());
+        out.writeNamedWriteable(mvExpand.target());
         out.writeNamedWriteable(mvExpand.expanded());
     }
 
@@ -1033,13 +1045,13 @@ static void writeOrderBy(PlanStreamOutput out, OrderBy order) throws IOException
     }
 
     static Project readProject(PlanStreamInput in) throws IOException {
-        return new Project(Source.readFrom(in), in.readLogicalPlanNode(), readNamedExpressions(in));
+        return new Project(Source.readFrom(in), in.readLogicalPlanNode(), in.readNamedWriteableCollectionAsList(NamedExpression.class));
     }
 
     static void writeProject(PlanStreamOutput out, Project project) throws IOException {
         Source.EMPTY.writeTo(out);
         out.writeLogicalPlanNode(project.child());
-        writeNamedExpressions(out, project.projections());
+        out.writeNamedWriteableCollection(project.projections());
     }
 
     static TopN readTopN(PlanStreamInput in) throws IOException {
@@ -1058,26 +1070,6 @@ static void writeTopN(PlanStreamOutput out, TopN topN) throws IOException {
         out.writeExpression(topN.limit());
     }
 
-    //
-    // -- Attributes
-    //
-
-    private static List<NamedExpression> readNamedExpressions(PlanStreamInput in) throws IOException {
-        return in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readNamedExpression));
-    }
-
-    static void writeNamedExpressions(PlanStreamOutput out, List<? extends NamedExpression> namedExpressions) throws IOException {
-        out.writeCollection(namedExpressions, writerFromPlanWriter(PlanStreamOutput::writeNamedExpression));
-    }
-
-    private static List<Alias> readAliases(PlanStreamInput in) throws IOException {
-        return in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readAlias));
-    }
-
-    static void writeAliases(PlanStreamOutput out, List<Alias> aliases) throws IOException {
-        out.writeCollection(aliases, writerFromPlanWriter(PlanNamedTypes::writeAlias));
-    }
-
     // -- BinaryComparison
 
     static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) throws IOException {
@@ -1683,28 +1675,6 @@ static void writeMvConcat(PlanStreamOutput out, MvConcat fn) throws IOException
         out.writeExpression(fn.right());
     }
 
-    // -- NamedExpressions
-
-    static Alias readAlias(PlanStreamInput in) throws IOException {
-        return new Alias(
-            Source.readFrom(in),
-            in.readString(),
-            in.readOptionalString(),
-            in.readNamed(Expression.class),
-            NameId.readFrom(in),
-            in.readBoolean()
-        );
-    }
-
-    static void writeAlias(PlanStreamOutput out, Alias alias) throws IOException {
-        Source.EMPTY.writeTo(out);
-        out.writeString(alias.name());
-        out.writeOptionalString(alias.qualifier());
-        out.writeExpression(alias.child());
-        alias.id().writeTo(out);
-        out.writeBoolean(alias.synthetic());
-    }
-
     // -- Expressions (other)
 
     static Literal readLiteral(PlanStreamInput in) throws IOException {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
index e7f1fbd6e1460..0b671d6b90c7e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.xpack.esql.Column;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.NameId;
-import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanNamedReader;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader;
@@ -93,14 +92,11 @@ public PhysicalPlan readOptionalPhysicalPlanNode() throws IOException {
         return readOptionalNamed(PhysicalPlan.class);
     }
 
+    @Override
     public Expression readExpression() throws IOException {
         return readNamed(Expression.class);
     }
 
-    public NamedExpression readNamedExpression() throws IOException {
-        return readNamed(NamedExpression.class);
-    }
-
     public <T> T readNamed(Class<T> type) throws IOException {
         String name = readString();
         @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
index 05dc7ab919868..f7380588fbd77 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.esql.Column;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
@@ -35,7 +34,7 @@
  * A customized stream output used to serialize ESQL physical plan fragments. Complements stream
  * output with methods that write plan nodes, Attributes, Expressions, etc.
  */
-public final class PlanStreamOutput extends StreamOutput {
+public final class PlanStreamOutput extends StreamOutput implements org.elasticsearch.xpack.esql.core.util.PlanStreamOutput {
 
     /**
      * Cache of written blocks. We use an {@link IdentityHashMap} for this
@@ -94,14 +93,11 @@ public void writeOptionalPhysicalPlanNode(PhysicalPlan physicalPlan) throws IOEx
         }
     }
 
+    @Override
     public void writeExpression(Expression expression) throws IOException {
         writeNamed(Expression.class, expression);
     }
 
-    public void writeNamedExpression(NamedExpression namedExpression) throws IOException {
-        writeNamed(NamedExpression.class, namedExpression);
-    }
-
     public void writeOptionalExpression(Expression expression) throws IOException {
         if (expression == null) {
             writeBoolean(false);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
index 6059b61031d1e..4fdc0bdab5ade 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
@@ -56,6 +56,7 @@
 import org.elasticsearch.xpack.esql.action.RestEsqlGetAsyncResultAction;
 import org.elasticsearch.xpack.esql.action.RestEsqlQueryAction;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.index.IndexResolver;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator;
@@ -195,6 +196,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.addAll(EsField.getNamedWriteables());
         entries.addAll(Attribute.getNamedWriteables());
         entries.add(UnsupportedAttribute.ENTRY);  // TODO combine with above once these are in the same project
+        entries.addAll(NamedExpression.getNamedWriteables());
+        entries.add(UnsupportedAttribute.NAMED_EXPRESSION_ENTRY);
         return entries;
     }
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
index 6ef33b7ae5eb8..a614ff3c621f8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
@@ -118,6 +119,8 @@ public static NamedWriteableRegistry writableRegistry() {
         entries.addAll(EsField.getNamedWriteables());
         entries.addAll(Attribute.getNamedWriteables());
         entries.add(UnsupportedAttribute.ENTRY);
+        entries.addAll(NamedExpression.getNamedWriteables());
+        entries.add(UnsupportedAttribute.NAMED_EXPRESSION_ENTRY);
         return new NamedWriteableRegistry(entries);
     }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java
new file mode 100644
index 0000000000000..ce7aa789f89b1
--- /dev/null
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AliasTests.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.AbstractWireTestCase;
+import org.elasticsearch.xpack.esql.core.expression.Alias;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NameId;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.tree.SourceTests;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests;
+import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
+import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.in;
+
+public class AliasTests extends AbstractWireTestCase<Alias> {
+    @Override
+    protected Alias createTestInstance() {
+        Source source = SourceTests.randomSource();
+        String name = randomAlphaOfLength(5);
+        String qualifier = randomBoolean() ? null : randomAlphaOfLength(3);
+        // TODO better randomChild
+        Expression child = ReferenceAttributeTests.randomReferenceAttribute();
+        boolean synthetic = randomBoolean();
+        return new Alias(source, name, qualifier, child, new NameId(), synthetic);
+    }
+
+    @Override
+    protected Alias mutateInstance(Alias instance) throws IOException {
+        Source source = instance.source();
+        String name = instance.name();
+        String qualifier = instance.qualifier();
+        Expression child = instance.child();
+        boolean synthetic = instance.synthetic();
+        switch (between(0, 3)) {
+            case 0 -> name = randomAlphaOfLength(name.length() + 1);
+            case 1 -> qualifier = randomValueOtherThan(qualifier, () -> randomBoolean() ? null : randomAlphaOfLength(3));
+            case 2 -> child = randomValueOtherThan(child, ReferenceAttributeTests::randomReferenceAttribute);
+            case 3 -> synthetic = false == synthetic;
+        }
+        return new Alias(source, name, qualifier, child, instance.id(), synthetic);
+    }
+
+    @Override
+    protected Alias copyInstance(Alias instance, TransportVersion version) throws IOException {
+        return copyInstance(
+            instance,
+            getNamedWriteableRegistry(),
+            (out, v) -> new PlanStreamOutput(out, new PlanNameRegistry(), null).writeNamedWriteable(v),
+            in -> {
+                PlanStreamInput pin = new PlanStreamInput(in, new PlanNameRegistry(), in.namedWriteableRegistry(), null);
+                Alias deser = (Alias) pin.readNamedWriteable(NamedExpression.class);
+                assertThat(deser.id(), equalTo(pin.mapNameId(Long.parseLong(instance.id().toString()))));
+                return deser;
+            },
+            version
+        );
+    }
+
+    @Override
+    protected final NamedWriteableRegistry getNamedWriteableRegistry() {
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(NamedExpression.getNamedWriteables());
+        entries.addAll(Attribute.getNamedWriteables());
+        entries.add(UnsupportedAttribute.ENTRY);
+        entries.addAll(EsField.getNamedWriteables());
+        return new NamedWriteableRegistry(entries);
+    }
+}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
index 716d4fa1f5cce..31d1018bacc91 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ReferenceAttributeTests.java
@@ -14,8 +14,7 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 
 public class ReferenceAttributeTests extends AbstractAttributeTestCase<ReferenceAttribute> {
-    @Override
-    protected ReferenceAttribute create() {
+    public static ReferenceAttribute randomReferenceAttribute() {
         Source source = Source.EMPTY;
         String name = randomAlphaOfLength(5);
         DataType type = randomFrom(DataType.types());
@@ -25,6 +24,11 @@ protected ReferenceAttribute create() {
         return new ReferenceAttribute(source, name, type, qualifier, nullability, new NameId(), synthetic);
     }
 
+    @Override
+    protected ReferenceAttribute create() {
+        return randomReferenceAttribute();
+    }
+
     @Override
     protected ReferenceAttribute mutate(ReferenceAttribute instance) {
         Source source = instance.source();
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
index 55c3811f9a870..2abc70b4ecc88 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
@@ -296,17 +296,6 @@ public void testPowSimple() throws IOException {
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
     }
 
-    public void testAliasSimple() throws IOException {
-        var orig = new Alias(Source.EMPTY, "alias_name", field("a", DataType.LONG));
-        BytesStreamOutput bso = new BytesStreamOutput();
-        PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null);
-        PlanNamedTypes.writeAlias(out, orig);
-        var in = planStreamInput(bso);
-        var deser = PlanNamedTypes.readAlias(in);
-        EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser);
-        assertThat(deser.id(), equalTo(in.mapNameId(Long.parseLong(orig.id().toString()))));
-    }
-
     public void testLiteralSimple() throws IOException {
         var orig = new Literal(Source.EMPTY, 1, DataType.INTEGER);
         BytesStreamOutput bso = new BytesStreamOutput();

From 21ffdac4a747103751c258063a9e1e3547b18f46 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 5 Jun 2024 11:13:55 -0700
Subject: [PATCH 178/208] Patch ImmutableCollections for tests (#109271)

ImmutableCollections uses a seed, set early during JVM startup, which
affects the iteration order of collections. Although we do not want to
rely on the iteration order of Map and Set collections, bugs do
sometimes occur. In order to reproduce those bugs to fix them, it is
important the test seed for Elasticsearch matches the seed used in
ImmutableCollections.

Unfortunately ImmutableCollections is internal to the JDK, and the seed
used is private and final. This commit works around these limitations by
creating a patched version of ImmutableCollections which allows access
to the seed member. ESTestCase is then able to reflectively set the seed
at runtime based on the Elasticsearch seed.

Note that this only affects tests. ImmutableCollections remains is
unchanged for production code.

relates #94946
---
 .../internal/ElasticsearchTestBasePlugin.java | 26 +++++++++
 .../gradle/internal/MrjarPlugin.java          |  1 -
 settings.gradle                               |  3 +-
 .../bootstrap/BootstrapForTesting.java        |  1 +
 .../org/elasticsearch/test/ESTestCase.java    | 51 +++++++++++++---
 test/immutable-collections-patch/build.gradle | 49 ++++++++++++++++
 .../patch/ImmutableCollectionsPatcher.java    | 58 +++++++++++++++++++
 7 files changed, 180 insertions(+), 9 deletions(-)
 create mode 100644 test/immutable-collections-patch/build.gradle
 create mode 100644 test/immutable-collections-patch/src/main/java/org/elasticsearch/jdk/patch/ImmutableCollectionsPatcher.java

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
index ed2dfb577e038..d344b4694a5b5 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
@@ -30,6 +30,7 @@
 import org.gradle.api.tasks.testing.Test;
 
 import java.io.File;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.gradle.util.FileUtils.mkdirs;
@@ -100,6 +101,7 @@ public void execute(Task t) {
                 "-Xmx" + System.getProperty("tests.heap.size", "512m"),
                 "-Xms" + System.getProperty("tests.heap.size", "512m"),
                 "-Djava.security.manager=allow",
+                "--add-opens=java.base/java.util=ALL-UNNAMED",
                 // TODO: only open these for mockito when it is modularized
                 "--add-opens=java.base/java.security.cert=ALL-UNNAMED",
                 "--add-opens=java.base/java.nio.channels=ALL-UNNAMED",
@@ -199,5 +201,29 @@ public void execute(Task t) {
                 }
             });
         });
+        configureImmutableCollectionsPatch(project);
+    }
+
+    private void configureImmutableCollectionsPatch(Project project) {
+        String patchProject = ":test:immutable-collections-patch";
+        if (project.findProject(patchProject) == null) {
+            return; // build tests may not have this project, just skip
+        }
+        String configurationName = "immutableCollectionsPatch";
+        FileCollection patchedFileCollection = project.getConfigurations()
+            .create(configurationName, config -> config.setCanBeConsumed(false));
+        var deps = project.getDependencies();
+        deps.add(configurationName, deps.project(Map.of("path", patchProject, "configuration", "patch")));
+        project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
+            test.getInputs().files(patchedFileCollection);
+            test.systemProperty("tests.hackImmutableCollections", "true");
+            test.getJvmArgumentProviders()
+                .add(
+                    () -> List.of(
+                        "--patch-module=java.base=" + patchedFileCollection.getSingleFile() + "/java.base",
+                        "--add-opens=java.base/java.util=ALL-UNNAMED"
+                    )
+                );
+        });
     }
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java
index 16c286bfdd3f2..756d1ea48610b 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java
@@ -172,7 +172,6 @@ private void createTestTask(Project project, SourceSet sourceSet, int javaVersio
                 testTask.getJavaLauncher()
                     .set(javaToolchains.launcherFor(spec -> spec.getLanguageVersion().set(JavaLanguageVersion.of(javaVersion))));
             }
-
         });
 
         project.getTasks().named("check").configure(checkTask -> checkTask.dependsOn(testTaskProvider));
diff --git a/settings.gradle b/settings.gradle
index 6ed340b27da65..ef758a7205cd0 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -105,7 +105,8 @@ List projects = [
   'test:test-clusters',
   'test:x-content',
   'test:yaml-rest-runner',
-  'test:metadata-extractor'
+  'test:metadata-extractor',
+  'test:immutable-collections-patch'
 ]
 
 /**
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
index 8a3f36ebb1f8a..30623c6bafd6b 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
@@ -73,6 +73,7 @@ public class BootstrapForTesting {
     // without making things complex???
 
     static {
+
         // make sure java.io.tmpdir exists always (in case code uses it in a static initializer)
         Path javaTmpDir = PathUtils.get(
             Objects.requireNonNull(System.getProperty("java.io.tmpdir"), "please set ${java.io.tmpdir} in pom.xml")
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 42cc719a904cd..6920083f2a1a6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -79,6 +79,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.PathUtilsForTesting;
@@ -145,6 +146,7 @@
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
+import java.lang.invoke.MethodHandles;
 import java.math.BigInteger;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
@@ -257,8 +259,9 @@ public static void resetPortCounter() {
     private static final SetOnce<Boolean> WARN_SECURE_RANDOM_FIPS_NOT_DETERMINISTIC = new SetOnce<>();
 
     static {
+        Random random = initTestSeed();
         TEST_WORKER_VM_ID = System.getProperty(TEST_WORKER_SYS_PROPERTY, DEFAULT_TEST_WORKER_ID);
-        setTestSysProps();
+        setTestSysProps(random);
         // TODO: consolidate logging initialization for tests so it all occurs in logconfigurator
         LogConfigurator.loadLog4jPlugins();
         LogConfigurator.configureESLogging();
@@ -359,8 +362,46 @@ public void append(LogEvent event) {
         JAVA_ZONE_IDS = ZoneId.getAvailableZoneIds().stream().filter(unsupportedZoneIdsPredicate.negate()).sorted().toList();
     }
 
+    static Random initTestSeed() {
+        String inputSeed = System.getProperty("tests.seed");
+        long seed;
+        if (inputSeed == null) {
+            // when running tests in intellij, we don't have a seed. Setup the seed early here, before getting to RandomizedRunner,
+            // so that we can use it in ESTestCase static init
+            seed = System.nanoTime();
+            setTestSeed(Long.toHexString(seed));
+        } else {
+            String[] seedParts = inputSeed.split("[\\:]");
+            seed = Long.parseUnsignedLong(seedParts[0], 16);
+        }
+
+        if (Booleans.parseBoolean(System.getProperty("tests.hackImmutableCollections", "false"))) {
+            forceImmutableCollectionsSeed(seed);
+        }
+
+        return new Random(seed);
+    }
+
+    @SuppressForbidden(reason = "set tests.seed for intellij")
+    static void setTestSeed(String seed) {
+        System.setProperty("tests.seed", seed);
+    }
+
+    private static void forceImmutableCollectionsSeed(long seed) {
+        try {
+            MethodHandles.Lookup lookup = MethodHandles.lookup();
+            Class<?> collectionsClass = Class.forName("java.util.ImmutableCollections");
+            var salt32l = lookup.findStaticVarHandle(collectionsClass, "SALT32L", long.class);
+            var reverse = lookup.findStaticVarHandle(collectionsClass, "REVERSE", boolean.class);
+            salt32l.set(seed & 0xFFFF_FFFFL);
+            reverse.set((seed & 1) == 0);
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
     @SuppressForbidden(reason = "force log4j and netty sysprops")
-    private static void setTestSysProps() {
+    private static void setTestSysProps(Random random) {
         System.setProperty("log4j.shutdownHookEnabled", "false");
         System.setProperty("log4j2.disable.jmx", "true");
 
@@ -377,11 +418,7 @@ private static void setTestSysProps() {
         System.setProperty("es.set.netty.runtime.available.processors", "false");
 
         // sometimes use the java.time date formatters
-        // we can't use randomBoolean here, the random context isn't set properly
-        // so read it directly from the test seed in an unfortunately hacky way
-        String testSeed = System.getProperty("tests.seed", "0");
-        boolean firstBit = (Integer.parseInt(testSeed.substring(testSeed.length() - 1), 16) & 1) == 1;
-        if (firstBit) {
+        if (random.nextBoolean()) {
             System.setProperty("es.datetime.java_time_parsers", "true");
         }
     }
diff --git a/test/immutable-collections-patch/build.gradle b/test/immutable-collections-patch/build.gradle
new file mode 100644
index 0000000000000..2d42215b3e02c
--- /dev/null
+++ b/test/immutable-collections-patch/build.gradle
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import org.elasticsearch.gradle.OS
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.internal.info.BuildParams
+
+apply plugin: 'elasticsearch.java'
+
+configurations {
+  patch
+}
+
+dependencies {
+  implementation 'org.ow2.asm:asm:9.7'
+  implementation 'org.ow2.asm:asm-tree:9.7'
+}
+
+def outputDir = layout.buildDirectory.dir("jdk-patches")
+def generatePatch = tasks.register("generatePatch", JavaExec)
+generatePatch.configure {
+  dependsOn tasks.named("compileJava")
+  inputs.property("java-home-set", BuildParams.getIsRuntimeJavaHomeSet())
+  inputs.property("java-version", BuildParams.runtimeJavaVersion)
+  outputs.dir(outputDir)
+
+  classpath = sourceSets.main.runtimeClasspath
+  mainClass = 'org.elasticsearch.jdk.patch.ImmutableCollectionsPatcher'
+  if (BuildParams.getIsRuntimeJavaHomeSet()) {
+    executable = "${BuildParams.runtimeJavaHome}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '')
+  } else {
+    javaLauncher = javaToolchains.launcherFor {
+      languageVersion = JavaLanguageVersion.of(BuildParams.runtimeJavaVersion.majorVersion)
+      vendor = VersionProperties.bundledJdkVendor == "openjdk" ?
+        JvmVendorSpec.ORACLE :
+        JvmVendorSpec.matching(VersionProperties.bundledJdkVendor)
+    }
+  }
+  doFirst {
+    args outputDir.get().getAsFile().toString()
+  }
+}
+
+artifacts.add("patch", generatePatch);
diff --git a/test/immutable-collections-patch/src/main/java/org/elasticsearch/jdk/patch/ImmutableCollectionsPatcher.java b/test/immutable-collections-patch/src/main/java/org/elasticsearch/jdk/patch/ImmutableCollectionsPatcher.java
new file mode 100644
index 0000000000000..b98df1b3d2e17
--- /dev/null
+++ b/test/immutable-collections-patch/src/main/java/org/elasticsearch/jdk/patch/ImmutableCollectionsPatcher.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.jdk.patch;
+
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.FieldVisitor;
+import org.objectweb.asm.Opcodes;
+
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * Loads ImmutableCollections.class from the current jdk and writes it out
+ * as a public class with SALT32L and REVERSE as public, non-final static fields.
+ *
+ * By exposing ImmutableCollections, tests run with this patched version can
+ * hook in the existing test seed to ensure consistent iteration of immutable collections.
+ * Note that the consistency is for <i>reproducing</i> dependencies on iteration
+ * order, so that the code can be fixed.
+ */
+public class ImmutableCollectionsPatcher {
+    private static final String CLASSFILE = "java.base/java/util/ImmutableCollections.class";
+
+    public static void main(String[] args) throws Exception {
+        Path outputDir = Paths.get(args[0]);
+        byte[] originalClassFile = Files.readAllBytes(Paths.get(URI.create("jrt:/" + CLASSFILE)));
+
+        ClassReader classReader = new ClassReader(originalClassFile);
+        ClassWriter classWriter = new ClassWriter(classReader, 0);
+        classReader.accept(new ClassVisitor(Opcodes.ASM9, classWriter) {
+            @Override
+            public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
+                super.visit(version, Opcodes.ACC_PUBLIC, name, signature, superName, interfaces);
+            }
+
+            @Override
+            public FieldVisitor visitField(int access, String name, String descriptor, String signature, Object value) {
+                if (name.equals("SALT32L") || name.equals("REVERSE")) {
+                    access = Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC;
+                }
+                return super.visitField(access, name, descriptor, signature, value);
+            }
+        }, 0);
+        Path outputFile = outputDir.resolve(CLASSFILE);
+        Files.createDirectories(outputFile.getParent());
+        Files.write(outputFile, classWriter.toByteArray());
+    }
+}

From 02a6c831e17869eb99187924dc1a80f758d2ff70 Mon Sep 17 00:00:00 2001
From: Lorenzo Verardo <loryok05@gmail.com>
Date: Wed, 5 Jun 2024 19:51:07 +0100
Subject: [PATCH 179/208] Limit the value in prefix query (#108537)

Reuse the setting index.max_regex_length for the max length in a prefix query.

Closes #108486
---
 docs/changelog/108537.yaml                    |  6 +++
 docs/reference/index-modules.asciidoc         |  2 +-
 .../rest-api-spec/test/search/30_limits.yml   | 28 +++++++++++++
 .../search/simple/SimpleSearchIT.java         | 40 +++++++++++++++++++
 .../index/query/PrefixQueryBuilder.java       | 15 +++++++
 5 files changed, 90 insertions(+), 1 deletion(-)
 create mode 100644 docs/changelog/108537.yaml

diff --git a/docs/changelog/108537.yaml b/docs/changelog/108537.yaml
new file mode 100644
index 0000000000000..1c0228a71d449
--- /dev/null
+++ b/docs/changelog/108537.yaml
@@ -0,0 +1,6 @@
+pr: 108537
+summary: Limit the value in prefix query
+area: Search
+type: enhancement
+issues:
+ - 108486
diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index e826956440497..40b4ff4bb9dc8 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -304,7 +304,7 @@ are ignored for this index.
 [[index-max-regex-length]]
  `index.max_regex_length`::
 
-    The maximum length of regex that can be used in Regexp Query.
+    The maximum length of value that can be used in `regexp` or `prefix` query.
     Defaults to `1000`.
 
 [[index-query-default-field]]
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/30_limits.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/30_limits.yml
index bea52c22e151f..f14614a820176 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/30_limits.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/30_limits.yml
@@ -161,3 +161,31 @@ setup:
                      ]))|\\[([^\\[\\]\\r\\\\]|\\\\.)*\\](?:(?:\\r\\n)?[\\t])*))*\\>(?:(?:\\r\\n)?[ \\t])*)(?:,\\s*( |
                      \".\\[\\]]))|\"(?:[^\\\"\\r\\\\]|\\\\.|(?:(?:\\r\\n)?[\\t]))*\"(?:(?:\\r\\n)?[ \\t])*)(?:\\.(?:( |
                      \\[\"()<>@,;:\\\\\".\\[\\]]))|\"(?:[^\\\"\\r\\\\]|\\\\.|(?:(?:\\r\\n)?[\\t]))*\"(?:(?:\\r\\n)?[\\t/"
+
+---
+"Prefix length limit":
+
+  - requires:
+      cluster_features: "gte_v8.15.0"
+      reason: "Limit for value in prefix query was introduced in 8.15"
+
+  - do:
+      catch: /The length of prefix \[1110\] used in the Prefix Query request has exceeded the allowed maximum of \[1000\]\. This maximum can be set by changing the \[index.max_regex_length\] index level setting\./
+      search:
+        rest_total_hits_as_int: true
+        index: test_1
+        body:
+          query:
+            prefix:
+              foo: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+                    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java
index 543f45b58279e..dd8cf5e527055 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java
@@ -490,6 +490,46 @@ public void testTooLongRegexInRegexpQuery() throws Exception {
         );
     }
 
+    public void testTooLongPrefixInPrefixQuery() throws Exception {
+        createIndex("idx");
+
+        // Ensure the field `num` exists in the mapping
+        client().admin()
+            .indices()
+            .preparePutMapping("idx")
+            .setSource("{\"properties\":{\"num\":{\"type\":\"keyword\"}}}", XContentType.JSON)
+            .get();
+
+        // Index a simple document to ensure the field `num` is in the index
+        indexRandom(true, prepareIndex("idx").setSource("{\"num\":\"test\"}", XContentType.JSON));
+
+        int defaultMaxRegexLength = IndexSettings.MAX_REGEX_LENGTH_SETTING.get(Settings.EMPTY);
+        StringBuilder prefix = new StringBuilder(defaultMaxRegexLength);
+
+        while (prefix.length() <= defaultMaxRegexLength) {
+            prefix.append("a");
+        }
+
+        SearchPhaseExecutionException e = expectThrows(
+            SearchPhaseExecutionException.class,
+            () -> client().prepareSearch("idx").setQuery(QueryBuilders.prefixQuery("num", prefix.toString())).get()
+        );
+        assertThat(
+            e.getRootCause().getMessage(),
+            containsString(
+                "The length of prefix ["
+                    + prefix.length()
+                    + "] used in the Prefix Query request has exceeded "
+                    + "the allowed maximum of ["
+                    + defaultMaxRegexLength
+                    + "]. "
+                    + "This maximum can be set by changing the ["
+                    + IndexSettings.MAX_REGEX_LENGTH_SETTING.getKey()
+                    + "] index level setting."
+            )
+        );
+    }
+
     public void testStrictlyCountRequest() throws Exception {
         createIndex("test_count_1");
         indexRandom(
diff --git a/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
index 5042ab358a96c..e64a424e86052 100644
--- a/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.ConstantFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.support.QueryParsers;
@@ -209,6 +210,20 @@ protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) throw
 
     @Override
     protected Query doToQuery(SearchExecutionContext context) throws IOException {
+        final int maxAllowedRegexLength = context.getIndexSettings().getMaxRegexLength();
+        if (value.length() > maxAllowedRegexLength) {
+            throw new IllegalArgumentException(
+                "The length of prefix ["
+                    + value.length()
+                    + "] used in the Prefix Query request has exceeded "
+                    + "the allowed maximum of ["
+                    + maxAllowedRegexLength
+                    + "]. "
+                    + "This maximum can be set by changing the ["
+                    + IndexSettings.MAX_REGEX_LENGTH_SETTING.getKey()
+                    + "] index level setting."
+            );
+        }
         MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite, null, LoggingDeprecationHandler.INSTANCE);
 
         MappedFieldType fieldType = context.getFieldType(fieldName);

From f8291f8e8308e8e55177455de3ba9f695e2c8f4b Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Thu, 6 Jun 2024 04:52:38 +1000
Subject: [PATCH 180/208] Forward port release notes for v8.14.0 (#109403)

---
 .../reference/migration/migrate_8_14.asciidoc |  49 ++-
 docs/reference/release-notes/8.14.0.asciidoc  | 346 +++++++++++++++++-
 .../release-notes/highlights.asciidoc         |  50 ++-
 3 files changed, 414 insertions(+), 31 deletions(-)

diff --git a/docs/reference/migration/migrate_8_14.asciidoc b/docs/reference/migration/migrate_8_14.asciidoc
index bdff8bef246b5..2e6cd439ebed0 100644
--- a/docs/reference/migration/migrate_8_14.asciidoc
+++ b/docs/reference/migration/migrate_8_14.asciidoc
@@ -21,8 +21,25 @@ and prevent them from operating normally.
 Before upgrading to 8.14, review these changes and take the described steps
 to mitigate the impact.
 
+
+There are no notable breaking changes in {es} 8.14.
+But there are some less critical breaking changes.
+
 [discrete]
-[[breaking-changes-8.14-0]]
+[[breaking_814_rest_api_changes]]
+==== REST API changes
+
+[[prevent_dls_fls_if_replication_assigned]]
+.Prevent DLS/FLS if `replication` is assigned
+[%collapsible]
+====
+*Details* +
+For cross-cluster API keys, {es} no longer allows specifying document-level security (DLS) or field-level security (FLS) in the `search` field, if `replication` is also specified. {es} likewise blocks the use of any existing cross-cluster API keys that meet this condition.
+
+*Impact* +
+Remove any document-level security (DLS) or field-level security (FLS) definitions from the `search` field for cross-cluster API keys that also have a `replication` field, or create two separate cross-cluster API keys, one for search and one for replication.
+====
+
 
 [discrete]
 [[breaking_814_dls_changes]]
@@ -41,3 +58,33 @@ When Document Level Security (DLS) is applied to the validate query API with the
 *Impact* +
 If needed, test workflows with DLS enabled to ensure that the stricter security rules do not impact your application.
 ====
+
+
+[discrete]
+[[deprecated-8.14]]
+=== Deprecations
+
+The following functionality has been deprecated in {es} 8.14
+and will be removed in a future version.
+While this won't have an immediate impact on your applications,
+we strongly encourage you to take the described steps to update your code
+after upgrading to 8.14.
+
+To find out if you are using any deprecated functionality,
+enable <<deprecation-logging, deprecation logging>>.
+
+[discrete]
+[[deprecations_814_mapping]]
+==== Mapping deprecations
+
+[[deprecate_allowing_fields_in_scenarios_where_it_ignored]]
+.Deprecate allowing `fields` in scenarios where it is ignored
+[%collapsible]
+====
+*Details* +
+The following mapped types have always ignored `fields` when using multi-fields. This deprecation makes this clearer and we will completely disallow `fields` for these mapped types in the future.
+
+*Impact* +
+In the future, `join`, `aggregate_metric_double`, and `constant_keyword`, will all disallow supplying `fields` as a parameter in the mapping.
+====
+
diff --git a/docs/reference/release-notes/8.14.0.asciidoc b/docs/reference/release-notes/8.14.0.asciidoc
index a203c983927cd..42f2f86a123ed 100644
--- a/docs/reference/release-notes/8.14.0.asciidoc
+++ b/docs/reference/release-notes/8.14.0.asciidoc
@@ -1,8 +1,350 @@
 [[release-notes-8.14.0]]
 == {es} version 8.14.0
 
-coming[8.14.0]
-
 Also see <<breaking-changes-8.14,Breaking changes in 8.14>>.
 
+[[breaking-8.14.0]]
+[float]
+=== Breaking changes
+
+Security::
+* Prevent DLS/FLS if `replication` is assigned {es-pull}108600[#108600]
+* Apply stricter Document Level Security (DLS) rules for the validate query API with the rewrite parameter {es-pull}105709[#105709]
+* Apply stricter Document Level Security (DLS) rules for terms aggregations when min_doc_count is set to 0 {es-pull}105714[#105714]
+
+[[bug-8.14.0]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Cross check livedocs for terms aggs when index access control list is non-null {es-pull}105714[#105714]
+* ESQL: Enable VALUES agg for datetime {es-pull}107016[#107016]
+* Fix IOOBE in TTest aggregation when using filters {es-pull}109034[#109034]
+* Validate stats formatting in standard `InternalStats` constructor {es-pull}107678[#107678] (issue: {es-issue}107671[#107671])
+
+Application::
+* [Bugfix] Connector API - fix status serialisation issue in termquery {es-pull}108365[#108365]
+* [Connector API] Fix bug with filtering validation toXContent {es-pull}107467[#107467]
+* [Connector API] Fix bug with parsing *_doc_count nullable fields {es-pull}108854[#108854]
+* [Connector API] Fix bug with with wrong target index for access control sync {es-pull}109097[#109097]
+
+Authorization::
+* Users with monitor privileges can access async_search/status endpoint even when setting keep_alive {es-pull}107383[#107383]
+
+CAT APIs::
+* Fix numeric sorts in `_cat/nodes` {es-pull}106189[#106189] (issue: {es-issue}48070[#48070])
+
+CCR::
+* Add ?master_timeout query parameter to ccr apis {es-pull}105168[#105168]
+
+CRUD::
+* Fix `noop_update_total` is not being updated when using the `_bulk` {es-pull}105745[#105745] (issue: {es-issue}105742[#105742])
+* Use correct system index bulk executor {es-pull}106150[#106150]
+
+Cluster Coordination::
+* Fix support for infinite `?master_timeout` {es-pull}107050[#107050]
+
+Data streams::
+* Add non-indexed fields to ecs templates {es-pull}106714[#106714]
+* Fix bulk NPE when retrying failure redirect after cluster block {es-pull}107598[#107598]
+* Improve error message when rolling over DS alias {es-pull}106708[#106708] (issue: {es-issue}106137[#106137])
+* Only skip deleting a downsampled index if downsampling is in progress as part of DSL retention {es-pull}109020[#109020]
+
+Downsampling::
+* Fix downsample action request serialization {es-pull}106919[#106919] (issue: {es-issue}106917[#106917])
+
+EQL::
+* Use #addWithoutBreaking when adding a negative number of bytes to the circuit breaker in `SequenceMatcher` {es-pull}107655[#107655]
+
+ES|QL::
+* ESQL: Allow reusing BUCKET grouping expressions in aggs {es-pull}107578[#107578]
+* ESQL: Disable quoting in FROM command {es-pull}108431[#108431]
+* ESQL: Fix MV_DEDUPE when using data from an index {es-pull}107577[#107577] (issue: {es-issue}104745[#104745])
+* ESQL: Fix error message when failing to resolve aggregate groupings {es-pull}108101[#108101] (issue: {es-issue}108053[#108053])
+* ESQL: Fix treating all fields as MV in COUNT pushdown {es-pull}106720[#106720]
+* ESQL: Re-enable logical dependency check {es-pull}105860[#105860]
+* ESQL: median, count and `count_distinct` over constants {es-pull}107414[#107414] (issues: {es-issue}105248[#105248], {es-issue}104900[#104900])
+* ES|QL fix no-length substring with supplementary (4-byte) character {es-pull}107183[#107183]
+* ES|QL: Fix usage of IN operator with TEXT fields {es-pull}106654[#106654] (issue: {es-issue}105379[#105379])
+* ES|QL: Improve support for TEXT fields in functions {es-pull}106810[#106810]
+* Fix docs generation of signatures for variadic functions {es-pull}107865[#107865]
+* [ESQL] Mark `date_diff` as requiring all three arguments {es-pull}108834[#108834] (issue: {es-issue}108383[#108383])
+
+Health::
+* Don't stop checking if the `HealthNode` persistent task is present {es-pull}105449[#105449] (issue: {es-issue}98926[#98926])
+* Health monitor concurrency fixes {es-pull}105674[#105674] (issue: {es-issue}105065[#105065])
+
+Highlighting::
+* Check preTags and postTags params for empty values {es-pull}106396[#106396] (issue: {es-issue}69009[#69009])
+* added fix for inconsistent text trimming in Unified Highlighter {es-pull}99961[#99961] (issue: {es-issue}101803[#101803])
+
+Infra/CLI::
+* Workaround G1 bug for JDK 22 and 22.0.1 {es-pull}108571[#108571]
+
+Infra/Core::
+* Add a check for the same feature being declared regular and historical {es-pull}106285[#106285]
+* Fix `AffixSetting.exists` to include secure settings {es-pull}106745[#106745]
+* Fix regression in get index settings (human=true) where the version was not displayed in human-readable format {es-pull}107447[#107447]
+* Nativeaccess: try to load all located libsystemds {es-pull}108238[#108238] (issue: {es-issue}107878[#107878])
+* Update several references to `IndexVersion.toString` to use `toReleaseVersion` {es-pull}107828[#107828] (issue: {es-issue}107821[#107821])
+* Update several references to `TransportVersion.toString` to use `toReleaseVersion` {es-pull}107902[#107902]
+
+Infra/Logging::
+* Log when update AffixSetting using addAffixMapUpdateConsumer {es-pull}97072[#97072]
+
+Infra/Node Lifecycle::
+* Consider `ShardRouting` roles when calculating shard copies in shutdown status {es-pull}106063[#106063]
+* Wait indefintely for http connections on shutdown by default {es-pull}106511[#106511]
+
+Infra/Scripting::
+* Guard against a null scorer in painless execute {es-pull}109048[#109048] (issue: {es-issue}43541[#43541])
+* Painless: Apply true regex limit factor with FIND and MATCH operation {es-pull}105670[#105670]
+
+Ingest Node::
+* Catching `StackOverflowErrors` from bad regexes in `GsubProcessor` {es-pull}106851[#106851]
+* Fix `uri_parts` processor behaviour for missing extensions {es-pull}105689[#105689] (issue: {es-issue}105612[#105612])
+* Remove leading is_ prefix from Enterprise geoip docs {es-pull}108518[#108518]
+* Slightly better geoip `databaseType` validation {es-pull}106889[#106889]
+
+License::
+* Fix lingering license warning header {es-pull}108031[#108031] (issue: {es-issue}107573[#107573])
+
+Machine Learning::
+* Fix NPE in ML assignment notifier {es-pull}107312[#107312]
+* Fix `startOffset` must be non-negative error in XLMRoBERTa tokenizer {es-pull}107891[#107891] (issue: {es-issue}104626[#104626])
+* Fix the position of spike, dip and distribution changes bucket when the sibling aggregation includes empty buckets {es-pull}106472[#106472]
+* Make OpenAI embeddings parser more flexible {es-pull}106808[#106808]
+
+Mapping::
+* Dedupe terms in terms queries {es-pull}106381[#106381]
+* Extend support of `allowedFields` to `getMatchingFieldNames` and `getAllFields` {es-pull}106862[#106862]
+* Fix for raw mapping merge of fields named "properties" {es-pull}108867[#108867] (issue: {es-issue}108866[#108866])
+* Handle infinity during synthetic source construction for scaled float field {es-pull}107494[#107494] (issue: {es-issue}107101[#107101])
+* Handle pass-through subfields with deep nesting {es-pull}106767[#106767]
+* Wrap "Pattern too complex" exception into an `IllegalArgumentException` {es-pull}109173[#109173]
+
+Network::
+* Fix HTTP corner-case response leaks {es-pull}105617[#105617]
+
+Search::
+* Add `internalClusterTest` for and fix leak in `ExpandSearchPhase` {es-pull}108562[#108562] (issue: {es-issue}108369[#108369])
+* Avoid attempting to load the same empty field twice in fetch phase {es-pull}107551[#107551]
+* Bugfix: Disable eager loading `BitSetFilterCache` on Indexing Nodes {es-pull}105791[#105791]
+* Cross-cluster painless/execute actions should check permissions only on target remote cluster {es-pull}105360[#105360]
+* Fix error 500 on invalid `ParentIdQuery` {es-pull}105693[#105693] (issue: {es-issue}105366[#105366])
+* Fix range queries for float/half_float fields when bounds are out of type's range {es-pull}106691[#106691]
+* Fixing NPE when requesting [_none_] for `stored_fields` {es-pull}104711[#104711]
+* Fork when handling remote field-caps responses {es-pull}107370[#107370]
+* Handle parallel calls to `createWeight` when profiling is on {es-pull}108041[#108041] (issues: {es-issue}104131[#104131], {es-issue}104235[#104235])
+* Harden field-caps request dispatcher {es-pull}108736[#108736]
+* Replace `UnsupportedOperationException` with `IllegalArgumentException` for non-existing columns {es-pull}107038[#107038]
+* Unable to retrieve multiple stored field values {es-pull}106575[#106575]
+* Validate `model_id` is required when using the `learning_to_rank` rescorer {es-pull}107743[#107743]
+
+Security::
+* Disable validate when rewrite parameter is sent and the index access control list is non-null {es-pull}105709[#105709]
+* Fix field caps and field level security {es-pull}106731[#106731]
+
+Snapshot/Restore::
+* Fix double-pausing shard snapshot {es-pull}109148[#109148] (issue: {es-issue}109143[#109143])
+* Treat 404 as empty register in `AzureBlobStore` {es-pull}108900[#108900] (issue: {es-issue}108504[#108504])
+* `SharedBlobCacheService.maybeFetchRegion` should use `computeCacheFileRegionSize` {es-pull}106685[#106685]
+
+TSDB::
+* Flip dynamic mapping condition when create tsid {es-pull}105636[#105636]
+
+Transform::
+* Consolidate permissions checks {es-pull}106413[#106413] (issue: {es-issue}105794[#105794])
+* Disable PIT for remote clusters {es-pull}107969[#107969]
+* Make force-stopping the transform always remove persistent task from cluster state {es-pull}106989[#106989] (issue: {es-issue}106811[#106811])
+* Only trigger action once per thread {es-pull}107232[#107232] (issue: {es-issue}107215[#107215])
+* [Transform] Auto retry Transform start {es-pull}106243[#106243]
+
+Vector Search::
+* Fix multithreading copies in lib vec {es-pull}108802[#108802]
+* [8.14] Fix multithreading copies in lib vec {es-pull}108810[#108810]
+
+[[deprecation-8.14.0]]
+[float]
+=== Deprecations
+
+Mapping::
+* Deprecate allowing `fields` in scenarios where it is ignored {es-pull}106031[#106031]
+
+[[enhancement-8.14.0]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Add a `PriorityQueue` backed by `BigArrays` {es-pull}106361[#106361]
+* All new `shard_seed` parameter for `random_sampler` agg {es-pull}104830[#104830]
+
+Allocation::
+* Add allocation stats {es-pull}105894[#105894]
+* Add index forecasts to /_cat/allocation output {es-pull}97561[#97561]
+
+Application::
+* [Profiling] Add TopN Functions API {es-pull}106860[#106860]
+* [Profiling] Allow to override index settings {es-pull}106172[#106172]
+* [Profiling] Speed up serialization of flamegraph {es-pull}105779[#105779]
+
+Authentication::
+* Support Profile Activate with JWTs with client authn {es-pull}105439[#105439] (issue: {es-issue}105342[#105342])
+
+Authorization::
+* Allow users to get status of own async search tasks {es-pull}106638[#106638]
+* [Security Solution] Add `read` permission for third party agent indices for `kibana_system` {es-pull}107046[#107046]
+
+Data streams::
+* Add data stream lifecycle to kibana reporting template {es-pull}106259[#106259]
+
+ES|QL::
+* Add ES|QL Locate function {es-pull}106899[#106899] (issue: {es-issue}106818[#106818])
+* Add ES|QL signum function {es-pull}106866[#106866]
+* Add status for enrich operator {es-pull}106036[#106036]
+* Add two new OGC functions ST_X and ST_Y {es-pull}105768[#105768]
+* Adjust array resizing in block builder {es-pull}106934[#106934]
+* Bulk loading enrich fields in ESQL {es-pull}106796[#106796]
+* ENRICH support for TEXT fields {es-pull}106435[#106435] (issue: {es-issue}105384[#105384])
+* ESQL: Add timers to many status results {es-pull}105421[#105421]
+* ESQL: Allow grouping key inside stats expressions {es-pull}106579[#106579]
+* ESQL: Introduce expression validation phase {es-pull}105477[#105477] (issue: {es-issue}105425[#105425])
+* ESQL: Log queries at debug level {es-pull}108257[#108257]
+* ESQL: Regex improvements {es-pull}106429[#106429]
+* ESQL: Sum of constants {es-pull}105454[#105454]
+* ESQL: Support ST_DISJOINT {es-pull}107007[#107007]
+* ESQL: Support partially folding CASE {es-pull}106094[#106094]
+* ESQL: Use faster field caps {es-pull}105067[#105067]
+* ESQL: extend BUCKET with spans {es-pull}107272[#107272]
+* ESQL: perform a reduction on the data node {es-pull}106516[#106516]
+* Expand support for ENRICH to full set supported by ES ingest processors {es-pull}106186[#106186] (issue: {es-issue}106162[#106162])
+* Introduce ordinal bytesref block {es-pull}106852[#106852] (issue: {es-issue}106387[#106387])
+* Leverage ordinals in enrich lookup {es-pull}107449[#107449]
+* Serialize big array blocks {es-pull}106373[#106373]
+* Serialize big array vectors {es-pull}106327[#106327]
+* Specialize serialization for `ArrayVectors` {es-pull}105893[#105893]
+* Specialize serialization of array blocks {es-pull}106102[#106102]
+* Speed up serialization of `BytesRefArray` {es-pull}106053[#106053]
+* Support ST_CONTAINS and ST_WITHIN {es-pull}106503[#106503]
+* Support ST_INTERSECTS between geometry column and other geometry or string {es-pull}104907[#104907] (issue: {es-issue}104874[#104874])
+
+Engine::
+* Add metric for calculating index flush time excluding waiting on locks {es-pull}107196[#107196]
+
+Highlighting::
+* Enable 'encoder' and 'tags_schema' highlighting settings at field level {es-pull}107224[#107224] (issue: {es-issue}94028[#94028])
+
+ILM+SLM::
+* Add a flag to re-enable writes on the final index after an ILM shrink action. {es-pull}107121[#107121] (issue: {es-issue}106599[#106599])
+
+Indices APIs::
+* Wait forever for `IndexTemplateRegistry` asset installation {es-pull}105985[#105985]
+
+Infra/CLI::
+* Enhance search tier GC options {es-pull}106526[#106526]
+* Increase KDF iteration count in `KeyStoreWrapper` {es-pull}107107[#107107]
+
+Infra/Core::
+* Add pluggable `BuildVersion` in `NodeMetadata` {es-pull}105757[#105757]
+
+Infra/Metrics::
+* Infrastructure for metering the update requests {es-pull}105063[#105063]
+* `DocumentParsingObserver` to accept an `indexName` to allow skipping system indices {es-pull}107041[#107041]
+
+Infra/Scripting::
+* String sha512() painless function {es-pull}99048[#99048] (issue: {es-issue}97691[#97691])
+
+Ingest Node::
+* Add support for the 'Anonymous IP' database to the geoip processor {es-pull}107287[#107287] (issue: {es-issue}90789[#90789])
+* Add support for the 'Enterprise' database to the geoip processor {es-pull}107377[#107377]
+* Adding `cache_stats` to geoip stats API {es-pull}107334[#107334]
+* Support data streams in enrich policy indices {es-pull}107291[#107291] (issue: {es-issue}98836[#98836])
+
+Machine Learning::
+* Add GET `_inference` for all inference endpoints {es-pull}107517[#107517]
+* Added a timeout parameter to the inference API {es-pull}107242[#107242]
+* Enable retrying on 500 error response from Cohere text embedding API {es-pull}105797[#105797]
+
+Mapping::
+* Make int8_hnsw our default index for new dense-vector fields {es-pull}106836[#106836]
+
+Ranking::
+* Add retrievers using the parser-only approach {es-pull}105470[#105470]
+
+Search::
+* Add Lucene spanish plural stemmer {es-pull}106952[#106952]
+* Add `modelId` and `modelText` to `KnnVectorQueryBuilder` {es-pull}106068[#106068]
+* Add a SIMD (Neon) optimised vector distance function for int8 {es-pull}106133[#106133]
+* Add transport version for search load autoscaling {es-pull}106377[#106377]
+* CCS with `minimize_roundtrips` performs incremental merges of each `SearchResponse` {es-pull}105781[#105781]
+* Track ongoing search tasks {es-pull}107129[#107129]
+
+Security::
+* Invalidating cross cluster API keys requires `manage_security` {es-pull}107411[#107411]
+* Show owner `realm_type` for returned API keys {es-pull}105629[#105629]
+
+Snapshot/Restore::
+* Add setting for max connections to S3 {es-pull}107533[#107533]
+* Distinguish different snapshot failures by log level {es-pull}105622[#105622]
+
+Stats::
+* (API+) CAT Nodes alias for shard header to match CAT Allocation {es-pull}105847[#105847]
+* Add total size in bytes to doc stats {es-pull}106840[#106840] (issue: {es-issue}97670[#97670])
+
+TSDB::
+* Improve short-circuiting downsample execution {es-pull}106563[#106563]
+* Support non-keyword dimensions as routing fields in TSDB {es-pull}105501[#105501]
+* Text fields are stored by default in TSDB indices {es-pull}106338[#106338] (issue: {es-issue}97039[#97039])
+
+Transform::
+* Check node shutdown before fail {es-pull}107358[#107358] (issue: {es-issue}100891[#100891])
+* Do not log error on node restart when the transform is already failed {es-pull}106171[#106171] (issue: {es-issue}106168[#106168])
+
+[[feature-8.14.0]]
+[float]
+=== New features
+
+Application::
+* Allow `typed_keys` for search application Search API {es-pull}108007[#108007]
+* [Connector API] Support cleaning up sync jobs when deleting a connector {es-pull}107253[#107253]
+
+ES|QL::
+* ESQL: Values aggregation function {es-pull}106065[#106065] (issue: {es-issue}103600[#103600])
+* ESQL: allow sorting by expressions and not only regular fields {es-pull}107158[#107158]
+* Support ES|QL requests through the `NodeClient::execute` {es-pull}106244[#106244]
+
+Indices APIs::
+* Add granular error list to alias action response {es-pull}106514[#106514] (issue: {es-issue}94478[#94478])
+
+Machine Learning::
+* Add Cohere rerank to `_inference` service {es-pull}106378[#106378]
+* Add support for Azure OpenAI embeddings to inference service {es-pull}107178[#107178]
+* Create default word based chunker {es-pull}107303[#107303]
+* Text structure endpoints to determine the structure of a list of messages and of an indexed field {es-pull}105660[#105660]
+
+Mapping::
+* Flatten object mappings when subobjects is false {es-pull}103542[#103542] (issues: {es-issue}99860[#99860], {es-issue}103497[#103497])
+
+Security::
+* Get and Query API Key with profile uid {es-pull}106531[#106531]
+
+Vector Search::
+* Adding support for hex-encoded byte vectors on knn-search {es-pull}105393[#105393]
+
+[[upgrade-8.14.0]]
+[float]
+=== Upgrades
+
+Infra/Core::
+* Upgrade jna to 5.12.1 {es-pull}105717[#105717]
+
+Ingest Node::
+* Updating the tika version to 2.9.1 in the ingest attachment plugin {es-pull}106315[#106315]
+
+Network::
+* Upgrade to Netty 4.1.107 {es-pull}105517[#105517]
+
+Packaging::
+* Update bundled JDK to Java 22 (again) {es-pull}108654[#108654]
+
 
diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc
index d39be07b0bf02..e6016fe438e24 100644
--- a/docs/reference/release-notes/highlights.asciidoc
+++ b/docs/reference/release-notes/highlights.asciidoc
@@ -44,38 +44,32 @@ faster indexing and similar retrieval latencies.
 
 {es-pull}103374[#103374]
 
-[discrete]
-[[query_phase_knn_supports_query_vector_builder]]
-=== Query phase KNN now supports query_vector_builder
-It is now possible to pass `model_text` and `model_id` within a `knn` query
-in the [query DSL](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-knn-query.html) to convert a text query into a dense vector and run the
-nearest neighbor query on it, instead of requiring the dense vector to be
-directly passed (within the `query_vector` parameter). Similar to the
-[top-level knn query](https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html) (executed in the DFS phase), it is possible to supply
-a `query_vector_builder` object containing a `text_embedding` object with
-`model_text` (the text query to be converted into a dense vector) and
-`model_id` (the identifier of a deployed model responsible for transforming
-the text query into a dense vector). Note that an embedding model with the
-referenced `model_id` needs to be [deployed on a ML node](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-deploy-models.html).
-in the cluster.
+// end::notable-highlights[]
 
-{es-pull}106068[#106068]
 
 [discrete]
-[[simd_neon_optimised_vector_distance_function_for_merging_int8_scalar_quantized_vectors_has_been_added]]
-=== A SIMD (Neon) optimised vector distance function for merging int8 Scalar Quantized vectors has been added
-An optimised int8 vector distance implementation for aarch64 has been added.
-This implementation is currently only used during merging.
-The vector distance implementation outperforms Lucene's Pamana Vector
-implementation for binary comparisons by approx 5x (depending on the number
-of dimensions). It does so by means of SIMD (Neon) intrinsics compiled into a
-separate native library and link by Panama's FFI. Comparisons are performed on
-off-heap mmap'ed vector data.
-Macro benchmarks, SO_Dense_Vector with scalar quantization enabled, shows
-significant improvements in merge times, approximately 3 times faster.
+[[new_custom_parser_for_iso_8601_datetimes]]
+=== New custom parser for ISO-8601 datetimes
+This introduces a new custom parser for ISO-8601 datetimes, for the `iso8601`, `strict_date_optional_time`, and
+`strict_date_optional_time_nanos` built-in date formats. This provides a performance improvement over the
+default Java date-time parsing. Whilst it maintains much of the same behaviour,
+the new parser does not accept nonsensical date-time strings that have multiple fractional seconds fields
+or multiple timezone specifiers. If the new parser fails to parse a string, it will then use the previous parser
+to parse it. If a large proportion of the input data consists of these invalid strings, this may cause
+a small performance degradation. If you wish to force the use of the old parsers regardless,
+set the JVM property `es.datetime.java_time_parsers=true` on all ES nodes.
 
-{es-pull}106133[#106133]
+{es-pull}106486[#106486]
 
-// end::notable-highlights[]
+[discrete]
+[[preview_support_for_connection_type_domain_isp_databases_in_geoip_processor]]
+=== Preview: Support for the 'Connection Type, 'Domain', and 'ISP' databases in the geoip processor
+As a Technical Preview, the {ref}/geoip-processor.html[`geoip`] processor can now use the commercial
+https://dev.maxmind.com/geoip/docs/databases/connection-type[GeoIP2 'Connection Type'],
+https://dev.maxmind.com/geoip/docs/databases/domain[GeoIP2 'Domain'],
+and
+https://dev.maxmind.com/geoip/docs/databases/isp[GeoIP2 'ISP']
+databases from MaxMind.
 
+{es-pull}108683[#108683]
 

From b19bab2e00b6911d7791dafa8730d8eb5a8429db Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 5 Jun 2024 15:22:11 -0400
Subject: [PATCH 181/208] ESQL: Fix a folding bug with MV_ZIP (#109404)

In the process of migrating all tests off of the deprecated
`AbstractScalarFunctionTestCase` I had to add some extra null tests to a
few functions. This discovered a bug in MV_EXPAND where the explicit
`MV_EXPAND(null, ["a", "b"])` would give different results then `a =
null; MV_EXPAND(a, ["a", "b"])`. This fixes that and completes the
migration off of `AbstractScalarFunctionTestCase`.
---
 .../src/main/resources/string.csv-spec        |  35 ++++
 .../function/scalar/multivalue/MvZip.java     |   7 +
 .../AbstractScalarFunctionTestCase.java       | 196 ------------------
 .../function/scalar/math/AbsTests.java        |  14 +-
 .../function/scalar/math/CeilTests.java       |  22 +-
 .../function/scalar/math/LogTests.java        |  14 +-
 .../function/scalar/math/PowTests.java        |  14 +-
 .../scalar/multivalue/MvZipTests.java         | 105 ++++++----
 8 files changed, 121 insertions(+), 286 deletions(-)
 delete mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java

diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
index 063b74584a28b..53d7d1fd0d352 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec
@@ -839,6 +839,41 @@ emp_no:integer | full_name:keyword | full_name_2:keyword | job_positions:keyword
 10005          | Kyoichi Maliniak  | Maliniak,Kyoichi    | null                                                                   | [-2.14,13.07]                 | [-2.14,13.07]
 ;
 
+mvZipLiteralNullDelim
+required_capability: mv_sort
+
+FROM employees
+| EVAL full_name = mv_zip(first_name, last_name, null)
+| KEEP emp_no, full_name
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | full_name:keyword
+10001          | null
+10002          | null
+10003          | null
+10004          | null
+10005          | null
+;
+
+mvZipLiteralLongDelim
+required_capability: mv_sort
+
+FROM employees
+| EVAL full_name = mv_zip(first_name, last_name, " words words words ")
+| KEEP emp_no, full_name
+| SORT emp_no
+| LIMIT 5;
+
+emp_no:integer | full_name:keyword
+10001          | Georgi words words words Facello
+10002          | Bezalel words words words Simmel
+10003          | Parto words words words Bamford
+10004          | Chirstian words words words Koblick
+10005          | Kyoichi words words words Maliniak
+;
+
+
 showTextFields
 from hosts | sort description, card, ip0, ip1 | where host == "beta" | keep host, host_group, description;
 ignoreOrder:true
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java
index 6298057b16013..4f42858cbedba 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
 import org.elasticsearch.xpack.esql.core.expression.function.OptionalArgument;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -93,6 +94,12 @@ public boolean foldable() {
         return mvLeft.foldable() && mvRight.foldable() && (delim == null || delim.foldable());
     }
 
+    @Override
+    public Nullability nullable() {
+        // Nullability.TRUE means if *any* parameter is null we return null. We're only null if the first two are null.
+        return Nullability.FALSE;
+    }
+
     @Override
     public EvalOperator.ExpressionEvaluator.Factory toEvaluator(
         Function<Expression, EvalOperator.ExpressionEvaluator.Factory> toEvaluator
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java
deleted file mode 100644
index c8fe9e536beea..0000000000000
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractScalarFunctionTestCase.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.expression.function.scalar;
-
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.expression.Literal;
-import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
-import org.elasticsearch.xpack.esql.core.tree.Location;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
-import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
-import org.hamcrest.Matcher;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import java.util.stream.Stream;
-
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- * Base class for function tests.
- * @deprecated extends from {@link AbstractFunctionTestCase} instead
- * and {@link AbstractFunctionTestCase#errorsForCasesWithoutExamples}.
- */
-@Deprecated
-public abstract class AbstractScalarFunctionTestCase extends AbstractFunctionTestCase {
-    /**
-     * Describe supported arguments. Build each argument with
-     * {@link #required} or {@link #optional}.
-     */
-    protected abstract List<ArgumentSpec> argSpec();
-
-    /**
-     * The data type that applying this function to arguments of this type should produce.
-     */
-    protected abstract DataType expectedType(List<DataType> argTypes);
-
-    /**
-     * Define a required argument.
-     */
-    protected final ArgumentSpec required(DataType... validTypes) {
-        return new ArgumentSpec(false, withNullAndSorted(validTypes));
-    }
-
-    /**
-     * Define an optional argument.
-     */
-    protected final ArgumentSpec optional(DataType... validTypes) {
-        return new ArgumentSpec(true, withNullAndSorted(validTypes));
-    }
-
-    private Set<DataType> withNullAndSorted(DataType[] validTypes) {
-        Set<DataType> realValidTypes = new LinkedHashSet<>();
-        Arrays.stream(validTypes).sorted(Comparator.comparing(DataType::nameUpper)).forEach(realValidTypes::add);
-        realValidTypes.add(DataType.NULL);
-        return realValidTypes;
-    }
-
-    public Set<DataType> sortedTypesSet(DataType[] validTypes, DataType... additionalTypes) {
-        Set<DataType> mergedSet = new LinkedHashSet<>();
-        Stream.concat(Stream.of(validTypes), Stream.of(additionalTypes))
-            .sorted(Comparator.comparing(DataType::nameUpper))
-            .forEach(mergedSet::add);
-        return mergedSet;
-    }
-
-    /**
-     * All integer types (long, int, short, byte). For passing to {@link #required} or {@link #optional}.
-     */
-    protected static DataType[] integers() {
-        return DataType.types().stream().filter(DataType::isInteger).toArray(DataType[]::new);
-    }
-
-    /**
-     * All rational types (double, float, whatever). For passing to {@link #required} or {@link #optional}.
-     */
-    protected static DataType[] rationals() {
-        return DataType.types().stream().filter(DataType::isRational).toArray(DataType[]::new);
-    }
-
-    /**
-     * All numeric types (integers and rationals.) For passing to {@link #required} or {@link #optional}.
-     */
-    protected static DataType[] numerics() {
-        return DataType.types().stream().filter(DataType::isNumeric).toArray(DataType[]::new);
-    }
-
-    protected final DataType[] representableNumerics() {
-        // TODO numeric should only include representable numbers but that is a change for a followup
-        return DataType.types().stream().filter(DataType::isNumeric).filter(EsqlDataTypes::isRepresentable).toArray(DataType[]::new);
-    }
-
-    protected record ArgumentSpec(boolean optional, Set<DataType> validTypes) {}
-
-    public final void testResolveType() {
-        List<ArgumentSpec> specs = argSpec();
-        for (int mutArg = 0; mutArg < specs.size(); mutArg++) {
-            for (DataType mutArgType : DataType.types()) {
-                List<Expression> args = new ArrayList<>(specs.size());
-                for (int arg = 0; arg < specs.size(); arg++) {
-                    if (mutArg == arg) {
-                        args.add(new Literal(new Source(Location.EMPTY, "arg" + arg), "", mutArgType));
-                    } else {
-                        args.add(new Literal(new Source(Location.EMPTY, "arg" + arg), "", specs.get(arg).validTypes.iterator().next()));
-                    }
-                }
-                assertResolution(specs, args, mutArg, mutArgType, specs.get(mutArg).validTypes.contains(mutArgType));
-                int optionalIdx = specs.size() - 1;
-                while (optionalIdx > 0 && specs.get(optionalIdx).optional()) {
-                    args.remove(optionalIdx--);
-                    assertResolution(
-                        specs,
-                        args,
-                        mutArg,
-                        mutArgType,
-                        args.size() <= mutArg || specs.get(mutArg).validTypes.contains(mutArgType)
-                    );
-                }
-            }
-        }
-    }
-
-    private void assertResolution(List<ArgumentSpec> specs, List<Expression> args, int mutArg, DataType mutArgType, boolean shouldBeValid) {
-        Expression exp = build(new Source(Location.EMPTY, "exp"), args);
-        logger.info("checking {} is {}", exp.nodeString(), shouldBeValid ? "valid" : "invalid");
-        if (shouldBeValid) {
-            assertResolveTypeValid(exp, expectedType(args.stream().map(Expression::dataType).toList()));
-            return;
-        }
-        Expression.TypeResolution resolution = exp.typeResolved();
-        assertFalse(exp.nodeString(), resolution.resolved());
-        assertThat(exp.nodeString(), resolution.message(), badTypeError(specs, mutArg, mutArgType));
-    }
-
-    protected Matcher<String> badTypeError(List<ArgumentSpec> spec, int badArgPosition, DataType badArgType) {
-        String ordinal = spec.size() == 1
-            ? ""
-            : TypeResolutions.ParamOrdinal.fromIndex(badArgPosition).name().toLowerCase(Locale.ROOT) + " ";
-        return equalTo(
-            ordinal
-                + "argument of [exp] must be ["
-                + expectedTypeName(spec.get(badArgPosition).validTypes())
-                + "], found value [arg"
-                + badArgPosition
-                + "] type ["
-                + badArgType.typeName()
-                + "]"
-        );
-    }
-
-    private String expectedTypeName(Set<DataType> validTypes) {
-        List<DataType> withoutNull = validTypes.stream().filter(t -> t != DataType.NULL).toList();
-        if (withoutNull.equals(Arrays.asList(strings()))) {
-            return "string";
-        }
-        if (withoutNull.equals(Arrays.asList(integers())) || withoutNull.equals(List.of(DataType.INTEGER))) {
-            return "integer";
-        }
-        if (withoutNull.equals(Arrays.asList(rationals()))) {
-            return "double";
-        }
-        if (withoutNull.equals(Arrays.asList(numerics())) || withoutNull.equals(Arrays.asList(representableNumerics()))) {
-            return "numeric";
-        }
-        if (withoutNull.equals(List.of(DataType.DATETIME))) {
-            return "datetime";
-        }
-        if (withoutNull.equals(List.of(DataType.IP))) {
-            return "ip";
-        }
-        List<DataType> negations = Stream.concat(Stream.of(numerics()), Stream.of(DataType.DATE_PERIOD, DataType.TIME_DURATION))
-            .sorted(Comparator.comparing(DataType::nameUpper))
-            .toList();
-        if (withoutNull.equals(negations)) {
-            return "numeric, date_period or time_duration";
-        }
-        if (validTypes.equals(Set.copyOf(Arrays.asList(representableTypes())))) {
-            return "representable";
-        }
-        if (validTypes.equals(Set.copyOf(Arrays.asList(representableNonSpatialTypes())))) {
-            return "representableNonSpatial";
-        }
-        throw new IllegalArgumentException("can't guess expected type for " + validTypes);
-    }
-}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java
index 5158fb9aad372..63642a01fa117 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsTests.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
 
 import java.math.BigInteger;
 import java.util.ArrayList;
@@ -23,7 +23,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class AbsTests extends AbstractScalarFunctionTestCase {
+public class AbsTests extends AbstractFunctionTestCase {
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
@@ -74,14 +74,4 @@ public AbsTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSu
     protected Expression build(Source source, List<Expression> args) {
         return new Abs(source, args.get(0));
     }
-
-    @Override
-    protected List<ArgumentSpec> argSpec() {
-        return List.of(required(numerics()));
-    }
-
-    @Override
-    protected DataType expectedType(List<DataType> argTypes) {
-        return argTypes.get(0);
-    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java
index f562ccbf0071b..735113c34ca1b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilTests.java
@@ -13,8 +13,8 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
 
 import java.math.BigInteger;
 import java.util.ArrayList;
@@ -23,7 +23,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class CeilTests extends AbstractScalarFunctionTestCase {
+public class CeilTests extends AbstractFunctionTestCase {
     public CeilTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
         this.testCase = testCaseSupplier.get();
     }
@@ -31,7 +31,7 @@ public CeilTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseS
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        suppliers.addAll(List.of(new TestCaseSupplier("large double value", () -> {
+        suppliers.addAll(List.of(new TestCaseSupplier("large double value", List.of(DataType.DOUBLE), () -> {
             double arg = 1 / randomDouble();
             return new TestCaseSupplier.TestCase(
                 List.of(new TestCaseSupplier.TypedData(arg, DataType.DOUBLE, "arg")),
@@ -39,7 +39,7 @@ public static Iterable<Object[]> parameters() {
                 DataType.DOUBLE,
                 equalTo(Math.ceil(arg))
             );
-        }), new TestCaseSupplier("integer value", () -> {
+        }), new TestCaseSupplier("integer value", List.of(DataType.INTEGER), () -> {
             int arg = randomInt();
             return new TestCaseSupplier.TestCase(
                 List.of(new TestCaseSupplier.TypedData(arg, DataType.INTEGER, "arg")),
@@ -47,7 +47,7 @@ public static Iterable<Object[]> parameters() {
                 DataType.INTEGER,
                 equalTo(arg)
             );
-        }), new TestCaseSupplier("long value", () -> {
+        }), new TestCaseSupplier("long value", List.of(DataType.LONG), () -> {
             long arg = randomLong();
             return new TestCaseSupplier.TestCase(
                 List.of(new TestCaseSupplier.TypedData(arg, DataType.LONG, "arg")),
@@ -66,17 +66,7 @@ public static Iterable<Object[]> parameters() {
             UNSIGNED_LONG_MAX,
             List.of()
         );
-        return parameterSuppliersFromTypedData(suppliers);
-    }
-
-    @Override
-    protected DataType expectedType(List<DataType> argTypes) {
-        return argTypes.get(0);
-    }
-
-    @Override
-    protected List<ArgumentSpec> argSpec() {
-        return List.of(required(numerics()));
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(false, suppliers)));
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
index a25fc66ab2d73..ce53fdbfc1851 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java
@@ -13,13 +13,13 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
 
 import java.util.List;
 import java.util.function.Supplier;
 
-public class LogTests extends AbstractScalarFunctionTestCase {
+public class LogTests extends AbstractFunctionTestCase {
     public LogTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
         this.testCase = testCaseSupplier.get();
     }
@@ -194,16 +194,6 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
     }
 
-    @Override
-    protected DataType expectedType(List<DataType> argTypes) {
-        return DataType.DOUBLE;
-    }
-
-    @Override
-    protected List<ArgumentSpec> argSpec() {
-        return List.of(optional(numerics()), required(numerics()));
-    }
-
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new Log(source, args.get(0), args.size() > 1 ? args.get(1) : null);
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
index 855e3070d442f..545e7c14ff2b2 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java
@@ -13,13 +13,13 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
 
 import java.util.List;
 import java.util.function.Supplier;
 
-public class PowTests extends AbstractScalarFunctionTestCase {
+public class PowTests extends AbstractFunctionTestCase {
     public PowTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
         this.testCase = testCaseSupplier.get();
     }
@@ -80,16 +80,6 @@ public static Iterable<Object[]> parameters() {
         return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
     }
 
-    @Override
-    protected DataType expectedType(List<DataType> argTypes) {
-        return DataType.DOUBLE;
-    }
-
-    @Override
-    protected List<ArgumentSpec> argSpec() {
-        return List.of(required(numerics()), required(numerics()));
-    }
-
     @Override
     protected Expression build(Source source, List<Expression> args) {
         return new Pow(source, args.get(0), args.get(1));
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java
index e465d72555e4e..30fe420f29960 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipTests.java
@@ -15,8 +15,8 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractScalarFunctionTestCase;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -25,59 +25,79 @@
 import static java.lang.Math.max;
 import static org.hamcrest.Matchers.equalTo;
 
-public class MvZipTests extends AbstractScalarFunctionTestCase {
+public class MvZipTests extends AbstractFunctionTestCase {
     public MvZipTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
         this.testCase = testCaseSupplier.get();
     }
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() {
+        // Note that any null is *not* null, so we explicitly test with nulls
         List<TestCaseSupplier> suppliers = new ArrayList<>();
-        suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD, DataType.KEYWORD, DataType.KEYWORD), () -> {
-            List<Object> left = randomList(1, 3, () -> randomLiteral(DataType.KEYWORD).value());
-            List<Object> right = randomList(1, 3, () -> randomLiteral(DataType.KEYWORD).value());
-            String delim = randomAlphaOfLengthBetween(1, 1);
+        for (DataType leftType : DataType.types()) {
+            if (leftType != DataType.NULL && DataType.isString(leftType) == false) {
+                continue;
+            }
+            for (DataType rightType : DataType.types()) {
+                if (rightType != DataType.NULL && DataType.isString(rightType) == false) {
+                    continue;
+                }
+                for (DataType delimType : DataType.types()) {
+                    if (delimType != DataType.NULL && DataType.isString(delimType) == false) {
+                        continue;
+                    }
+                    suppliers.add(supplier(leftType, rightType, delimType));
+                }
+                suppliers.add(supplier(leftType, rightType));
+            }
+        }
+
+        return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers));
+    }
+
+    private static TestCaseSupplier supplier(DataType leftType, DataType rightType, DataType delimType) {
+        return new TestCaseSupplier(List.of(leftType, rightType, delimType), () -> {
+            List<BytesRef> left = randomList(leftType);
+            List<BytesRef> right = randomList(rightType);
+            BytesRef delim = delimType == DataType.NULL ? null : new BytesRef(randomAlphaOfLength(1));
+
             List<BytesRef> expected = calculateExpected(left, right, delim);
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(left, DataType.KEYWORD, "mvLeft"),
-                    new TestCaseSupplier.TypedData(right, DataType.KEYWORD, "mvRight"),
-                    new TestCaseSupplier.TypedData(delim, DataType.KEYWORD, "delim")
+                    new TestCaseSupplier.TypedData(left, leftType, "mvLeft"),
+                    new TestCaseSupplier.TypedData(right, rightType, "mvRight"),
+                    new TestCaseSupplier.TypedData(delim, delimType, "delim")
                 ),
                 "MvZipEvaluator[leftField=Attribute[channel=0], rightField=Attribute[channel=1], delim=Attribute[channel=2]]",
                 DataType.KEYWORD,
-                equalTo(expected.size() == 1 ? expected.iterator().next() : expected)
+                equalTo(expected == null ? null : expected.size() == 1 ? expected.iterator().next() : expected)
             );
-        }));
+        });
+    }
 
-        suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT, DataType.TEXT, DataType.TEXT), () -> {
-            List<Object> left = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value());
-            List<Object> right = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value());
-            String delim = randomAlphaOfLengthBetween(1, 1);
-            List<BytesRef> expected = calculateExpected(left, right, delim);
+    private static TestCaseSupplier supplier(DataType leftType, DataType rightType) {
+        return new TestCaseSupplier(List.of(leftType, rightType), () -> {
+            List<BytesRef> left = randomList(leftType);
+            List<BytesRef> right = randomList(rightType);
+
+            List<BytesRef> expected = calculateExpected(left, right, new BytesRef(","));
             return new TestCaseSupplier.TestCase(
                 List.of(
-                    new TestCaseSupplier.TypedData(left, DataType.TEXT, "mvLeft"),
-                    new TestCaseSupplier.TypedData(right, DataType.TEXT, "mvRight"),
-                    new TestCaseSupplier.TypedData(delim, DataType.TEXT, "delim")
+                    new TestCaseSupplier.TypedData(left, leftType, "mvLeft"),
+                    new TestCaseSupplier.TypedData(right, rightType, "mvRight")
                 ),
-                "MvZipEvaluator[leftField=Attribute[channel=0], rightField=Attribute[channel=1], delim=Attribute[channel=2]]",
+                "MvZipEvaluator[leftField=Attribute[channel=0], rightField=Attribute[channel=1], delim=LiteralsEvaluator[lit=,]]",
                 DataType.KEYWORD,
-                equalTo(expected.size() == 1 ? expected.iterator().next() : expected)
+                equalTo(expected == null ? null : expected.size() == 1 ? expected.iterator().next() : expected)
             );
-        }));
-
-        return parameterSuppliersFromTypedData(suppliers);
+        });
     }
 
-    @Override
-    protected DataType expectedType(List<DataType> argTypes) {
-        return DataType.KEYWORD;
-    }
-
-    @Override
-    protected List<ArgumentSpec> argSpec() {
-        return List.of(required(strings()), required(strings()), optional(strings()));
+    private static List<BytesRef> randomList(DataType type) {
+        if (type == DataType.NULL) {
+            return null;
+        }
+        return randomList(1, 3, () -> new BytesRef(randomAlphaOfLength(5)));
     }
 
     @Override
@@ -85,27 +105,36 @@ protected Expression build(Source source, List<Expression> args) {
         return new MvZip(source, args.get(0), args.get(1), args.size() > 2 ? args.get(2) : null);
     }
 
-    private static List<BytesRef> calculateExpected(List<Object> left, List<Object> right, String delim) {
+    private static List<BytesRef> calculateExpected(List<BytesRef> left, List<BytesRef> right, BytesRef delim) {
+        if (delim == null) {
+            return null;
+        }
+        if (left == null) {
+            return right;
+        }
+        if (right == null) {
+            return left;
+        }
         List<BytesRef> expected = new ArrayList<>(max(left.size(), right.size()));
         int i = 0, j = 0;
         while (i < left.size() && j < right.size()) {
             BytesRefBuilder work = new BytesRefBuilder();
-            work.append((BytesRef) left.get(i));
-            work.append(new BytesRef(delim));
-            work.append((BytesRef) right.get(j));
+            work.append(left.get(i));
+            work.append(delim);
+            work.append(right.get(j));
             expected.add(work.get());
             i++;
             j++;
         }
         while (i < left.size()) {
             BytesRefBuilder work = new BytesRefBuilder();
-            work.append((BytesRef) left.get(i));
+            work.append(left.get(i));
             expected.add(work.get());
             i++;
         }
         while (j < right.size()) {
             BytesRefBuilder work = new BytesRefBuilder();
-            work.append((BytesRef) right.get(j));
+            work.append(right.get(j));
             expected.add(work.get());
             j++;
         }

From 7ea0d4be5480520962193032e52b354239b0ef3a Mon Sep 17 00:00:00 2001
From: James Baiera <james.baiera@gmail.com>
Date: Wed, 5 Jun 2024 16:51:22 -0400
Subject: [PATCH 182/208] Add support for failure stores in ILM (#108741)

Failure stores on a data stream will inherit the ILM policy of the parent data stream by default.
This PR adds logic to ensure failure stores are properly accounted for in data stream related
ILM operations.
---
 .../cluster/metadata/DataStream.java          |  34 +++
 .../cluster/metadata/DataStreamTests.java     |  81 ++++++
 .../MetadataDeleteIndexServiceTests.java      |  70 +++++
 .../ilm/CheckNotDataStreamWriteIndexStep.java |   7 +-
 .../xpack/core/ilm/DeleteStep.java            |  18 +-
 .../ReplaceDataStreamBackingIndexStep.java    |  13 +-
 .../xpack/core/ilm/RolloverStep.java          |  18 +-
 .../core/ilm/WaitForActiveShardsStep.java     |  13 +-
 .../core/ilm/WaitForRolloverReadyStep.java    |  32 ++-
 .../CheckNoDataStreamWriteIndexStepTests.java |  52 +++-
 .../xpack/core/ilm/DeleteStepTests.java       | 251 +++++++++++++++++-
 ...eplaceDataStreamBackingIndexStepTests.java | 116 ++++++--
 .../xpack/core/ilm/RolloverStepTests.java     |  48 +++-
 .../core/ilm/WaitForActiveShardsTests.java    |  46 +++-
 .../ilm/WaitForRolloverReadyStepTests.java    |  70 ++++-
 15 files changed, 782 insertions(+), 87 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
index ae01b7c064749..bf1d9462ab89f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
@@ -636,6 +636,40 @@ public DataStream replaceBackingIndex(Index existingBackingIndex, Index newBacki
             .build();
     }
 
+    /**
+     * Replaces the specified failure store index with a new index and returns a new {@code DataStream} instance with
+     * the modified backing indices. An {@code IllegalArgumentException} is thrown if the index to be replaced
+     * is not a failure store index for this data stream or if it is the {@code DataStream}'s failure store write index.
+     *
+     * @param existingFailureIndex the failure store index to be replaced
+     * @param newFailureIndex      the new index that will be part of the {@code DataStream}
+     * @return new {@code DataStream} instance with failure store indices that contain replacement index instead of the specified
+     * existing index.
+     */
+    public DataStream replaceFailureStoreIndex(Index existingFailureIndex, Index newFailureIndex) {
+        List<Index> currentFailureIndices = new ArrayList<>(failureIndices.indices);
+        int failureIndexPosition = currentFailureIndices.indexOf(existingFailureIndex);
+        if (failureIndexPosition == -1) {
+            throw new IllegalArgumentException(
+                String.format(Locale.ROOT, "index [%s] is not part of data stream [%s] failure store", existingFailureIndex.getName(), name)
+            );
+        }
+        if (failureIndices.indices.size() == (failureIndexPosition + 1)) {
+            throw new IllegalArgumentException(
+                String.format(
+                    Locale.ROOT,
+                    "cannot replace failure index [%s] of data stream [%s] because it is the failure store write index",
+                    existingFailureIndex.getName(),
+                    name
+                )
+            );
+        }
+        currentFailureIndices.set(failureIndexPosition, newFailureIndex);
+        return copy().setFailureIndices(this.failureIndices.copy().setIndices(currentFailureIndices).build())
+            .setGeneration(generation + 1)
+            .build();
+    }
+
     /**
      * Adds the specified index as a backing index and returns a new {@code DataStream} instance with the new combination
      * of backing indices.
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
index 1c1f6b314fa70..0277855db9c4c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
@@ -731,6 +731,15 @@ public void testReplaceBackingIndexThrowsExceptionIfIndexNotPartOfDataStream() {
         expectThrows(IllegalArgumentException.class, () -> original.replaceBackingIndex(standaloneIndex, newBackingIndex));
     }
 
+    public void testReplaceBackingIndexThrowsExceptionIfIndexPartOfFailureStore() {
+        DataStream original = createRandomDataStream();
+        int indexToReplace = randomIntBetween(1, original.getFailureIndices().getIndices().size() - 1) - 1;
+
+        Index failureIndex = original.getFailureIndices().getIndices().get(indexToReplace);
+        Index newBackingIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random()));
+        expectThrows(IllegalArgumentException.class, () -> original.replaceBackingIndex(failureIndex, newBackingIndex));
+    }
+
     public void testReplaceBackingIndexThrowsExceptionIfReplacingWriteIndex() {
         int numBackingIndices = randomIntBetween(2, 32);
         int writeIndexPosition = numBackingIndices - 1;
@@ -761,6 +770,78 @@ public void testReplaceBackingIndexThrowsExceptionIfReplacingWriteIndex() {
         );
     }
 
+    public void testReplaceFailureIndex() {
+        DataStream original = createRandomDataStream();
+        int indexToReplace = randomIntBetween(1, original.getFailureIndices().getIndices().size() - 1) - 1;
+
+        Index newFailureIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random()));
+        DataStream updated = original.replaceFailureStoreIndex(
+            original.getFailureIndices().getIndices().get(indexToReplace),
+            newFailureIndex
+        );
+        assertThat(updated.getName(), equalTo(original.getName()));
+        assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1));
+        assertThat(updated.getFailureIndices().getIndices().size(), equalTo(original.getFailureIndices().getIndices().size()));
+        assertThat(updated.getFailureIndices().getIndices().get(indexToReplace), equalTo(newFailureIndex));
+
+        for (int i = 0; i < original.getFailureIndices().getIndices().size(); i++) {
+            if (i != indexToReplace) {
+                assertThat(updated.getFailureIndices().getIndices().get(i), equalTo(original.getFailureIndices().getIndices().get(i)));
+            }
+        }
+    }
+
+    public void testReplaceFailureIndexThrowsExceptionIfIndexNotPartOfDataStream() {
+        DataStream original = createRandomDataStream();
+
+        Index standaloneIndex = new Index("index-foo", UUIDs.randomBase64UUID(random()));
+        Index newFailureIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random()));
+        expectThrows(IllegalArgumentException.class, () -> original.replaceFailureStoreIndex(standaloneIndex, newFailureIndex));
+    }
+
+    public void testReplaceFailureIndexThrowsExceptionIfIndexPartOfBackingIndices() {
+        DataStream original = createRandomDataStream();
+        int indexToReplace = randomIntBetween(1, original.getIndices().size() - 1) - 1;
+
+        Index backingIndex = original.getIndices().get(indexToReplace);
+        Index newFailureIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random()));
+        expectThrows(IllegalArgumentException.class, () -> original.replaceFailureStoreIndex(backingIndex, newFailureIndex));
+    }
+
+    public void testReplaceFailureIndexThrowsExceptionIfReplacingWriteIndex() {
+        int numFailureIndices = randomIntBetween(2, 32);
+        int writeIndexPosition = numFailureIndices - 1;
+        String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        long ts = System.currentTimeMillis();
+
+        List<Index> indices = new ArrayList<>(1);
+        indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts), UUIDs.randomBase64UUID(random())));
+
+        List<Index> failureIndices = new ArrayList<>(numFailureIndices);
+        for (int i = 1; i <= numFailureIndices; i++) {
+            failureIndices.add(new Index(DataStream.getDefaultFailureStoreName(dataStreamName, i, ts), UUIDs.randomBase64UUID(random())));
+        }
+        int generation = randomBoolean() ? numFailureIndices : numFailureIndices + randomIntBetween(1, 5);
+        DataStream original = newInstance(dataStreamName, indices, generation, null, false, null, failureIndices);
+
+        Index newBackingIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random()));
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> original.replaceFailureStoreIndex(failureIndices.get(writeIndexPosition), newBackingIndex)
+        );
+        assertThat(
+            e.getMessage(),
+            equalTo(
+                String.format(
+                    Locale.ROOT,
+                    "cannot replace failure index [%s] of data stream [%s] because it is the failure store write index",
+                    failureIndices.get(writeIndexPosition).getName(),
+                    dataStreamName
+                )
+            )
+        );
+    }
+
     public void testSnapshot() {
         var preSnapshotDataStream = DataStreamTestHelper.randomInstance();
         var indicesToRemove = randomSubsetOf(preSnapshotDataStream.getIndices());
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java
index b7bd54eef2c70..344acb7a8ff40 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java
@@ -274,6 +274,76 @@ public void testDeleteCurrentWriteIndexForDataStream() {
         );
     }
 
+    public void testDeleteMultipleFailureIndexForDataStream() {
+        int numBackingIndices = randomIntBetween(3, 5);
+        int numBackingIndicesToDelete = randomIntBetween(2, numBackingIndices - 1);
+        String dataStreamName = randomAlphaOfLength(6).toLowerCase(Locale.ROOT);
+        long ts = System.currentTimeMillis();
+        ClusterState before = DataStreamTestHelper.getClusterStateWithDataStreams(
+            List.of(new Tuple<>(dataStreamName, numBackingIndices)),
+            List.of(),
+            ts,
+            Settings.EMPTY,
+            1,
+            false,
+            true
+        );
+
+        List<Integer> indexNumbersToDelete = randomSubsetOf(
+            numBackingIndicesToDelete,
+            IntStream.rangeClosed(1, numBackingIndices - 1).boxed().toList()
+        );
+
+        Set<Index> indicesToDelete = new HashSet<>();
+        for (int k : indexNumbersToDelete) {
+            indicesToDelete.add(before.metadata().index(DataStream.getDefaultFailureStoreName(dataStreamName, k, ts)).getIndex());
+        }
+        ClusterState after = MetadataDeleteIndexService.deleteIndices(before, indicesToDelete, Settings.EMPTY);
+
+        DataStream dataStream = after.metadata().dataStreams().get(dataStreamName);
+        assertThat(dataStream, notNullValue());
+        assertThat(dataStream.getFailureIndices().getIndices().size(), equalTo(numBackingIndices - indexNumbersToDelete.size()));
+        for (Index i : indicesToDelete) {
+            assertThat(after.metadata().getIndices().get(i.getName()), nullValue());
+            assertFalse(dataStream.getFailureIndices().getIndices().contains(i));
+        }
+        assertThat(after.metadata().getIndices().size(), equalTo((2 * numBackingIndices) - indexNumbersToDelete.size()));
+    }
+
+    public void testDeleteCurrentWriteFailureIndexForDataStream() {
+        int numBackingIndices = randomIntBetween(1, 5);
+        String dataStreamName = randomAlphaOfLength(6).toLowerCase(Locale.ROOT);
+        long ts = System.currentTimeMillis();
+        ClusterState before = DataStreamTestHelper.getClusterStateWithDataStreams(
+            List.of(new Tuple<>(dataStreamName, numBackingIndices)),
+            List.of(),
+            ts,
+            Settings.EMPTY,
+            1,
+            false,
+            true
+        );
+
+        Index indexToDelete = before.metadata()
+            .index(DataStream.getDefaultFailureStoreName(dataStreamName, numBackingIndices, ts))
+            .getIndex();
+        Exception e = expectThrows(
+            IllegalArgumentException.class,
+            () -> MetadataDeleteIndexService.deleteIndices(before, Set.of(indexToDelete), Settings.EMPTY)
+        );
+
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "index ["
+                    + indexToDelete.getName()
+                    + "] is the failure store write index for data stream ["
+                    + dataStreamName
+                    + "] and cannot be deleted"
+            )
+        );
+    }
+
     private ClusterState clusterState(String index) {
         IndexMetadata indexMetadata = IndexMetadata.builder(index)
             .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersionUtils.randomVersion(random())))
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java
index e716a18738bca..28b04bc9614bb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java
@@ -62,14 +62,15 @@ public Result isConditionMet(Index index, ClusterState clusterState) {
         assert indexAbstraction != null : "invalid cluster metadata. index [" + indexName + "] was not found";
         DataStream dataStream = indexAbstraction.getParentDataStream();
         if (dataStream != null) {
-            assert dataStream.getWriteIndex() != null : dataStream.getName() + " has no write index";
-            if (dataStream.getWriteIndex().equals(index)) {
+            boolean isFailureStoreWriteIndex = index.equals(dataStream.getFailureStoreWriteIndex());
+            if (isFailureStoreWriteIndex || dataStream.getWriteIndex().equals(index)) {
                 String errorMessage = String.format(
                     Locale.ROOT,
-                    "index [%s] is the write index for data stream [%s], pausing "
+                    "index [%s] is the%s write index for data stream [%s], pausing "
                         + "ILM execution of lifecycle [%s] until this index is no longer the write index for the data stream via manual or "
                         + "automated rollover",
                     indexName,
+                    isFailureStoreWriteIndex ? " failure store" : "",
                     dataStream.getName(),
                     policyName
                 );
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java
index ba6b6f9366c61..282f32da28a6b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.Index;
 
 import java.util.Locale;
 
@@ -40,13 +41,17 @@ public void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState cu
         DataStream dataStream = indexAbstraction.getParentDataStream();
 
         if (dataStream != null) {
-            assert dataStream.getWriteIndex() != null : dataStream.getName() + " has no write index";
+            Index failureStoreWriteIndex = dataStream.getFailureStoreWriteIndex();
+            boolean isFailureStoreWriteIndex = failureStoreWriteIndex != null && indexName.equals(failureStoreWriteIndex.getName());
 
             // using index name equality across this if/else branch as the UUID of the index might change via restoring a data stream
             // with one index from snapshot
-            if (dataStream.getIndices().size() == 1 && dataStream.getWriteIndex().getName().equals(indexName)) {
-                // This is the last index in the data stream, the entire stream
-                // needs to be deleted, because we can't have an empty data stream
+            if (dataStream.getIndices().size() == 1
+                && isFailureStoreWriteIndex == false
+                && dataStream.getWriteIndex().getName().equals(indexName)) {
+                // This is the last backing index in the data stream, and it's being deleted because the policy doesn't have a rollover
+                // phase. The entire stream needs to be deleted, because we can't have an empty list of data stream backing indices.
+                // We do this even if there are multiple failure store indices because otherwise we would never delete the index.
                 DeleteDataStreamAction.Request deleteReq = new DeleteDataStreamAction.Request(new String[] { dataStream.getName() });
                 getClient().execute(
                     DeleteDataStreamAction.INSTANCE,
@@ -54,13 +59,14 @@ public void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState cu
                     listener.delegateFailureAndWrap((l, response) -> l.onResponse(null))
                 );
                 return;
-            } else if (dataStream.getWriteIndex().getName().equals(indexName)) {
+            } else if (isFailureStoreWriteIndex || dataStream.getWriteIndex().getName().equals(indexName)) {
                 String errorMessage = String.format(
                     Locale.ROOT,
-                    "index [%s] is the write index for data stream [%s]. "
+                    "index [%s] is the%s write index for data stream [%s]. "
                         + "stopping execution of lifecycle [%s] as a data stream's write index cannot be deleted. manually rolling over the"
                         + " index will resume the execution of the policy as the index will not be the data stream's write index anymore",
                     indexName,
+                    isFailureStoreWriteIndex ? " failure store" : "",
                     dataStream.getName(),
                     policyName
                 );
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java
index 9de08c8693a12..3962768e94212 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java
@@ -86,14 +86,15 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
             throw new IllegalStateException(errorMessage);
         }
 
-        assert dataStream.getWriteIndex() != null : dataStream.getName() + " has no write index";
-        if (dataStream.getWriteIndex().equals(index)) {
+        boolean isFailureStoreWriteIndex = index.equals(dataStream.getFailureStoreWriteIndex());
+        if (isFailureStoreWriteIndex || dataStream.getWriteIndex().equals(index)) {
             String errorMessage = String.format(
                 Locale.ROOT,
-                "index [%s] is the write index for data stream [%s], pausing "
+                "index [%s] is the%s write index for data stream [%s], pausing "
                     + "ILM execution of lifecycle [%s] until this index is no longer the write index for the data stream via manual or "
                     + "automated rollover",
                 originalIndex,
+                isFailureStoreWriteIndex ? " failure store" : "",
                 dataStream.getName(),
                 policyName
             );
@@ -114,8 +115,10 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
             throw new IllegalStateException(errorMessage);
         }
 
-        Metadata.Builder newMetaData = Metadata.builder(clusterState.getMetadata())
-            .put(dataStream.replaceBackingIndex(index, targetIndexMetadata.getIndex()));
+        DataStream updatedDataStream = dataStream.isFailureStoreIndex(originalIndex)
+            ? dataStream.replaceFailureStoreIndex(index, targetIndexMetadata.getIndex())
+            : dataStream.replaceBackingIndex(index, targetIndexMetadata.getIndex());
+        Metadata.Builder newMetaData = Metadata.builder(clusterState.getMetadata()).put(updatedDataStream);
         return ClusterState.builder(clusterState).metadata(newMetaData).build();
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java
index 26300f646d617..3e6c00eeadba4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
@@ -57,13 +58,16 @@ public void performAction(
         IndexAbstraction indexAbstraction = currentClusterState.metadata().getIndicesLookup().get(indexName);
         assert indexAbstraction != null : "expected the index " + indexName + " to exist in the lookup but it didn't";
         final String rolloverTarget;
+        final boolean targetFailureStore;
         DataStream dataStream = indexAbstraction.getParentDataStream();
         if (dataStream != null) {
-            assert dataStream.getWriteIndex() != null : "datastream " + dataStream.getName() + " has no write index";
-            if (dataStream.getWriteIndex().equals(indexMetadata.getIndex()) == false) {
+            boolean isFailureStoreWriteIndex = indexMetadata.getIndex().equals(dataStream.getFailureStoreWriteIndex());
+            targetFailureStore = dataStream.isFailureStoreIndex(indexMetadata.getIndex().getName());
+            if (isFailureStoreWriteIndex == false && dataStream.getWriteIndex().equals(indexMetadata.getIndex()) == false) {
                 logger.warn(
-                    "index [{}] is not the write index for data stream [{}]. skipping rollover for policy [{}]",
+                    "index [{}] is not the {}write index for data stream [{}]. skipping rollover for policy [{}]",
                     indexName,
+                    targetFailureStore ? "failure store " : "",
                     dataStream.getName(),
                     indexMetadata.getLifecyclePolicyName()
                 );
@@ -115,10 +119,18 @@ public void performAction(
             }
 
             rolloverTarget = rolloverAlias;
+            targetFailureStore = false;
         }
 
         // Calling rollover with no conditions will always roll over the index
         RolloverRequest rolloverRequest = new RolloverRequest(rolloverTarget, null).masterNodeTimeout(TimeValue.MAX_VALUE);
+        if (targetFailureStore) {
+            rolloverRequest.setIndicesOptions(
+                IndicesOptions.builder(rolloverRequest.indicesOptions())
+                    .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false))
+                    .build()
+            );
+        }
         // We don't wait for active shards when we perform the rollover because the
         // {@link org.elasticsearch.xpack.core.ilm.WaitForActiveShardsStep} step will do so
         rolloverRequest.setWaitForActiveShards(ActiveShardCount.NONE);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java
index b6cf8b0bdd663..71c99d7f21848 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java
@@ -30,7 +30,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetadata.parseIndexNameCounter;
 
 /**
- * After we performed the index rollover we wait for the the configured number of shards for the rolled over index (ie. newly created
+ * After we performed the index rollover we wait for the configured number of shards for the rolled over index (ie. newly created
  * index) to become available.
  */
 public class WaitForActiveShardsStep extends ClusterStateWaitStep {
@@ -84,10 +84,17 @@ public Result isConditionMet(Index index, ClusterState clusterState) {
         if (dataStream != null) {
             IndexAbstraction dataStreamAbstraction = metadata.getIndicesLookup().get(dataStream.getName());
             assert dataStreamAbstraction != null : dataStream.getName() + " datastream is not present in the metadata indices lookup";
-            if (dataStreamAbstraction.getWriteIndex() == null) {
+            // Determine which write index we care about right now:
+            final Index rolledIndex;
+            if (dataStream.isFailureStoreIndex(index.getName())) {
+                rolledIndex = dataStream.getFailureStoreWriteIndex();
+            } else {
+                rolledIndex = dataStream.getWriteIndex();
+            }
+            if (rolledIndex == null) {
                 return getErrorResultOnNullMetadata(getKey(), index);
             }
-            IndexMetadata rolledIndexMeta = metadata.index(dataStreamAbstraction.getWriteIndex());
+            IndexMetadata rolledIndexMeta = metadata.index(rolledIndex);
             rolledIndexName = rolledIndexMeta.getIndex().getName();
             waitForActiveShardsSettingValue = rolledIndexMeta.getSettings().get(IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS.getKey());
         } else {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
index acb36bd015e4b..7b751994222b1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.rollover.RolloverConditions;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
@@ -83,13 +84,16 @@ public void evaluateCondition(Metadata metadata, Index index, Listener listener,
         IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(index.getName());
         assert indexAbstraction != null : "invalid cluster metadata. index [" + index.getName() + "] was not found";
         final String rolloverTarget;
+        final boolean targetFailureStore;
         DataStream dataStream = indexAbstraction.getParentDataStream();
         if (dataStream != null) {
-            assert dataStream.getWriteIndex() != null : "datastream " + dataStream.getName() + " has no write index";
-            if (dataStream.getWriteIndex().equals(index) == false) {
+            targetFailureStore = dataStream.isFailureStoreIndex(index.getName());
+            boolean isFailureStoreWriteIndex = index.equals(dataStream.getFailureStoreWriteIndex());
+            if (isFailureStoreWriteIndex == false && dataStream.getWriteIndex().equals(index) == false) {
                 logger.warn(
-                    "index [{}] is not the write index for data stream [{}]. skipping rollover for policy [{}]",
+                    "index [{}] is not the {}write index for data stream [{}]. skipping rollover for policy [{}]",
                     index.getName(),
+                    targetFailureStore ? "failure store " : "",
                     dataStream.getName(),
                     metadata.index(index).getLifecyclePolicyName()
                 );
@@ -194,12 +198,18 @@ public void evaluateCondition(Metadata metadata, Index index, Listener listener,
             }
 
             rolloverTarget = rolloverAlias;
+            targetFailureStore = false;
         }
 
         // if we should only rollover if not empty, *and* if neither an explicit min_docs nor an explicit min_primary_shard_docs
         // has been specified on this policy, then inject a default min_docs: 1 condition so that we do not rollover empty indices
         boolean rolloverOnlyIfHasDocuments = LifecycleSettings.LIFECYCLE_ROLLOVER_ONLY_IF_HAS_DOCUMENTS_SETTING.get(metadata.settings());
-        RolloverRequest rolloverRequest = createRolloverRequest(rolloverTarget, masterTimeout, rolloverOnlyIfHasDocuments);
+        RolloverRequest rolloverRequest = createRolloverRequest(
+            rolloverTarget,
+            masterTimeout,
+            rolloverOnlyIfHasDocuments,
+            targetFailureStore
+        );
 
         getClient().admin().indices().rolloverIndex(rolloverRequest, ActionListener.wrap(response -> {
             final var conditionStatus = response.getConditionStatus();
@@ -226,10 +236,22 @@ public void evaluateCondition(Metadata metadata, Index index, Listener listener,
      * @return A RolloverRequest suitable for passing to {@code rolloverIndex(...) }.
      */
     // visible for testing
-    RolloverRequest createRolloverRequest(String rolloverTarget, TimeValue masterTimeout, boolean rolloverOnlyIfHasDocuments) {
+    RolloverRequest createRolloverRequest(
+        String rolloverTarget,
+        TimeValue masterTimeout,
+        boolean rolloverOnlyIfHasDocuments,
+        boolean targetFailureStore
+    ) {
         RolloverRequest rolloverRequest = new RolloverRequest(rolloverTarget, null).masterNodeTimeout(masterTimeout);
         rolloverRequest.dryRun(true);
         rolloverRequest.setConditions(applyDefaultConditions(conditions, rolloverOnlyIfHasDocuments));
+        if (targetFailureStore) {
+            rolloverRequest.setIndicesOptions(
+                IndicesOptions.builder(rolloverRequest.indicesOptions())
+                    .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false))
+                    .build()
+            );
+        }
         return rolloverRequest;
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java
index 33d571fbe8599..e0957239e33a8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java
@@ -65,29 +65,45 @@ public void testStepCompleteIfIndexIsNotPartOfDataStream() {
 
     public void testStepIncompleteIfIndexIsTheDataStreamWriteIndex() {
         String dataStreamName = randomAlphaOfLength(10);
-        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
+        long ts = System.currentTimeMillis();
+        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts);
+        String failureIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts);
         String policyName = "test-ilm-policy";
         IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
             .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
+        IndexMetadata failureIndexMetadata = IndexMetadata.builder(failureIndexName)
+            .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(emptyClusterState())
             .metadata(
-                Metadata.builder().put(indexMetadata, true).put(newInstance(dataStreamName, List.of(indexMetadata.getIndex()))).build()
+                Metadata.builder()
+                    .put(indexMetadata, true)
+                    .put(failureIndexMetadata, true)
+                    .put(newInstance(dataStreamName, List.of(indexMetadata.getIndex()), List.of(failureIndexMetadata.getIndex())))
+                    .build()
             )
             .build();
 
-        ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(indexMetadata.getIndex(), clusterState);
+        boolean useFailureStore = randomBoolean();
+        IndexMetadata indexToOperateOn = useFailureStore ? failureIndexMetadata : indexMetadata;
+        String expectedIndexName = indexToOperateOn.getIndex().getName();
+        ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(indexToOperateOn.getIndex(), clusterState);
         assertThat(result.isComplete(), is(false));
         SingleMessageFieldInfo info = (SingleMessageFieldInfo) result.getInfomationContext();
         assertThat(
             info.getMessage(),
             is(
                 "index ["
-                    + indexName
-                    + "] is the write index for data stream ["
+                    + expectedIndexName
+                    + "] is the "
+                    + (useFailureStore ? "failure store " : "")
+                    + "write index for data stream ["
                     + dataStreamName
                     + "], "
                     + "pausing ILM execution of lifecycle ["
@@ -100,33 +116,51 @@ public void testStepIncompleteIfIndexIsTheDataStreamWriteIndex() {
 
     public void testStepCompleteIfPartOfDataStreamButNotWriteIndex() {
         String dataStreamName = randomAlphaOfLength(10);
-        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
+        long ts = System.currentTimeMillis();
+        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts);
+        String failureIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts);
         String policyName = "test-ilm-policy";
         IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
             .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
+        IndexMetadata failureIndexMetadata = IndexMetadata.builder(failureIndexName)
+            .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
 
-        String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2);
+        String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2, ts);
+        String failureStoreWriteIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 2, ts);
         IndexMetadata writeIndexMetadata = IndexMetadata.builder(writeIndexName)
             .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
+        IndexMetadata failureStoreWriteIndexMetadata = IndexMetadata.builder(failureStoreWriteIndexName)
+            .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
 
         List<Index> backingIndices = List.of(indexMetadata.getIndex(), writeIndexMetadata.getIndex());
+        List<Index> failureIndices = List.of(failureIndexMetadata.getIndex(), failureStoreWriteIndexMetadata.getIndex());
         ClusterState clusterState = ClusterState.builder(emptyClusterState())
             .metadata(
                 Metadata.builder()
                     .put(indexMetadata, true)
                     .put(writeIndexMetadata, true)
-                    .put(newInstance(dataStreamName, backingIndices))
+                    .put(failureIndexMetadata, true)
+                    .put(failureStoreWriteIndexMetadata, true)
+                    .put(newInstance(dataStreamName, backingIndices, failureIndices))
                     .build()
             )
             .build();
 
-        ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(indexMetadata.getIndex(), clusterState);
+        boolean useFailureStore = randomBoolean();
+        IndexMetadata indexToOperateOn = useFailureStore ? failureIndexMetadata : indexMetadata;
+        ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(indexToOperateOn.getIndex(), clusterState);
         assertThat(result.isComplete(), is(true));
         assertThat(result.getInfomationContext(), is(nullValue()));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java
index 7445e82da3ecf..af4dc67d5dcbd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteStepTests.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.ClusterState;
@@ -130,10 +131,11 @@ public void testPerformActionCallsFailureListenerIfIndexIsTheDataStreamWriteInde
 
         String policyName = "test-ilm-policy";
         String dataStreamName = randomAlphaOfLength(10);
+        long ts = System.currentTimeMillis();
 
         IndexMetadata index1;
         {
-            String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
+            String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts);
             index1 = IndexMetadata.builder(indexName)
                 .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
                 .numberOfShards(randomIntBetween(1, 5))
@@ -142,25 +144,258 @@ public void testPerformActionCallsFailureListenerIfIndexIsTheDataStreamWriteInde
         }
         IndexMetadata sourceIndexMetadata;
         {
-
-            String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2);
+            String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2, ts);
             sourceIndexMetadata = IndexMetadata.builder(indexName)
                 .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
                 .numberOfShards(randomIntBetween(1, 5))
                 .numberOfReplicas(randomIntBetween(0, 5))
                 .build();
         }
+        IndexMetadata failureIndex1;
+        {
+            String indexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts);
+            failureIndex1 = IndexMetadata.builder(indexName)
+                .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+        }
+        IndexMetadata failureSourceIndexMetadata;
+        {
+            String indexName = DataStream.getDefaultFailureStoreName(dataStreamName, 2, ts);
+            failureSourceIndexMetadata = IndexMetadata.builder(indexName)
+                .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+        }
 
         DataStream dataStream = DataStreamTestHelper.newInstance(
             dataStreamName,
-            List.of(index1.getIndex(), sourceIndexMetadata.getIndex())
+            List.of(index1.getIndex(), sourceIndexMetadata.getIndex()),
+            List.of(failureIndex1.getIndex(), failureSourceIndexMetadata.getIndex())
+        );
+        ClusterState clusterState = ClusterState.builder(emptyClusterState())
+            .metadata(
+                Metadata.builder()
+                    .put(index1, false)
+                    .put(sourceIndexMetadata, false)
+                    .put(failureIndex1, false)
+                    .put(failureSourceIndexMetadata, false)
+                    .put(dataStream)
+                    .build()
+            )
+            .build();
+
+        AtomicBoolean listenerCalled = new AtomicBoolean(false);
+        final boolean useFailureStore = randomBoolean();
+        final IndexMetadata indexToOperateOn = useFailureStore ? failureSourceIndexMetadata : sourceIndexMetadata;
+        createRandomInstance().performDuringNoSnapshot(indexToOperateOn, clusterState, new ActionListener<>() {
+            @Override
+            public void onResponse(Void complete) {
+                listenerCalled.set(true);
+                fail("unexpected listener callback");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                listenerCalled.set(true);
+                assertThat(
+                    e.getMessage(),
+                    is(
+                        "index ["
+                            + indexToOperateOn.getIndex().getName()
+                            + "] is the "
+                            + (useFailureStore ? "failure store " : "")
+                            + "write index for data stream ["
+                            + dataStreamName
+                            + "]. stopping execution of lifecycle [test-ilm-policy] as a data stream's write index cannot be deleted. "
+                            + "manually rolling over the index will resume the execution of the policy as the index will not be the "
+                            + "data stream's write index anymore"
+                    )
+                );
+            }
+        });
+
+        assertThat(listenerCalled.get(), is(true));
+    }
+
+    public void testDeleteWorksIfWriteIndexIsTheOnlyIndexInDataStream() throws Exception {
+        String policyName = "test-ilm-policy";
+        String dataStreamName = randomAlphaOfLength(10);
+        long ts = System.currentTimeMillis();
+
+        // Single backing index
+        IndexMetadata index1;
+        {
+            String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts);
+            index1 = IndexMetadata.builder(indexName)
+                .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+        }
+
+        DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, List.of(index1.getIndex()), List.of());
+
+        ClusterState clusterState = ClusterState.builder(emptyClusterState())
+            .metadata(Metadata.builder().put(index1, false).put(dataStream).build())
+            .build();
+
+        Mockito.doAnswer(invocation -> {
+            DeleteDataStreamAction.Request request = (DeleteDataStreamAction.Request) invocation.getArguments()[1];
+            @SuppressWarnings("unchecked")
+            ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[2];
+            assertNotNull(request);
+            assertEquals(1, request.getNames().length);
+            assertEquals(dataStreamName, request.getNames()[0]);
+            listener.onResponse(null);
+            return null;
+        }).when(client).execute(any(), any(), any());
+
+        // Try on the normal data stream - It should delete the data stream
+        DeleteStep step = createRandomInstance();
+        PlainActionFuture.<Void, Exception>get(f -> step.performAction(index1, clusterState, null, f));
+
+        Mockito.verify(client, Mockito.only()).execute(any(), any(), any());
+        Mockito.verify(adminClient, Mockito.never()).indices();
+        Mockito.verify(indicesClient, Mockito.never()).delete(any(), any());
+    }
+
+    public void testDeleteWorksIfWriteIndexIsTheOnlyIndexInDataStreamWithFailureStore() throws Exception {
+        String policyName = "test-ilm-policy";
+        String dataStreamName = randomAlphaOfLength(10);
+        long ts = System.currentTimeMillis();
+
+        // Single backing index
+        IndexMetadata index1;
+        {
+            String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts);
+            index1 = IndexMetadata.builder(indexName)
+                .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+        }
+
+        // Multiple failure indices
+        IndexMetadata failureIndex1;
+        {
+            String indexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts);
+            failureIndex1 = IndexMetadata.builder(indexName)
+                .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+        }
+        IndexMetadata failureSourceIndexMetadata;
+        {
+            String indexName = DataStream.getDefaultFailureStoreName(dataStreamName, 2, ts);
+            failureSourceIndexMetadata = IndexMetadata.builder(indexName)
+                .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+        }
+
+        DataStream dataStreamWithFailureIndices = DataStreamTestHelper.newInstance(
+            dataStreamName,
+            List.of(index1.getIndex()),
+            List.of(failureIndex1.getIndex(), failureSourceIndexMetadata.getIndex())
         );
+
         ClusterState clusterState = ClusterState.builder(emptyClusterState())
-            .metadata(Metadata.builder().put(index1, false).put(sourceIndexMetadata, false).put(dataStream).build())
+            .metadata(
+                Metadata.builder()
+                    .put(index1, false)
+                    .put(failureIndex1, false)
+                    .put(failureSourceIndexMetadata, false)
+                    .put(dataStreamWithFailureIndices)
+                    .build()
+            )
+            .build();
+
+        Mockito.doAnswer(invocation -> {
+            DeleteDataStreamAction.Request request = (DeleteDataStreamAction.Request) invocation.getArguments()[1];
+            @SuppressWarnings("unchecked")
+            ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[2];
+            assertNotNull(request);
+            assertEquals(1, request.getNames().length);
+            assertEquals(dataStreamName, request.getNames()[0]);
+            listener.onResponse(null);
+            return null;
+        }).when(client).execute(any(), any(), any());
+
+        // Again, the deletion should work since the data stream would be fully deleted anyway if the failure store were disabled.
+        DeleteStep step = createRandomInstance();
+        PlainActionFuture.<Void, Exception>get(f -> step.performAction(index1, clusterState, null, f));
+
+        Mockito.verify(client, Mockito.only()).execute(any(), any(), any());
+        Mockito.verify(adminClient, Mockito.never()).indices();
+        Mockito.verify(indicesClient, Mockito.never()).delete(any(), any());
+    }
+
+    public void testDeletingFailureStoreWriteIndexOnDataStreamWithSingleBackingIndex() {
+        doThrow(
+            new IllegalStateException(
+                "the client must not be called in this test as we should fail in the step validation phase before we call the delete API"
+            )
+        ).when(indicesClient).delete(any(DeleteIndexRequest.class), anyActionListener());
+
+        String policyName = "test-ilm-policy";
+        String dataStreamName = randomAlphaOfLength(10);
+        long ts = System.currentTimeMillis();
+
+        // Single backing index
+        IndexMetadata index1;
+        {
+            String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts);
+            index1 = IndexMetadata.builder(indexName)
+                .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+        }
+
+        // Multiple failure indices
+        IndexMetadata failureIndex1;
+        {
+            String indexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts);
+            failureIndex1 = IndexMetadata.builder(indexName)
+                .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+        }
+        IndexMetadata failureSourceIndexMetadata;
+        {
+            String indexName = DataStream.getDefaultFailureStoreName(dataStreamName, 2, ts);
+            failureSourceIndexMetadata = IndexMetadata.builder(indexName)
+                .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+        }
+
+        DataStream dataStreamWithFailureIndices = DataStreamTestHelper.newInstance(
+            dataStreamName,
+            List.of(index1.getIndex()),
+            List.of(failureIndex1.getIndex(), failureSourceIndexMetadata.getIndex())
+        );
+
+        ClusterState clusterState = ClusterState.builder(emptyClusterState())
+            .metadata(
+                Metadata.builder()
+                    .put(index1, false)
+                    .put(failureIndex1, false)
+                    .put(failureSourceIndexMetadata, false)
+                    .put(dataStreamWithFailureIndices)
+                    .build()
+            )
             .build();
 
         AtomicBoolean listenerCalled = new AtomicBoolean(false);
-        createRandomInstance().performDuringNoSnapshot(sourceIndexMetadata, clusterState, new ActionListener<>() {
+        createRandomInstance().performDuringNoSnapshot(failureSourceIndexMetadata, clusterState, new ActionListener<>() {
             @Override
             public void onResponse(Void complete) {
                 listenerCalled.set(true);
@@ -174,8 +409,8 @@ public void onFailure(Exception e) {
                     e.getMessage(),
                     is(
                         "index ["
-                            + sourceIndexMetadata.getIndex().getName()
-                            + "] is the write index for data stream ["
+                            + failureSourceIndexMetadata.getIndex().getName()
+                            + "] is the failure store write index for data stream ["
                             + dataStreamName
                             + "]. stopping execution of lifecycle [test-ilm-policy] as a data stream's write index cannot be deleted. "
                             + "manually rolling over the index will resume the execution of the policy as the index will not be the "
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStepTests.java
index 2a49be703574b..a3318e68305c6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStepTests.java
@@ -68,55 +68,85 @@ public void testPerformActionThrowsExceptionIfIndexIsNotPartOfDataStream() {
 
     public void testPerformActionThrowsExceptionIfIndexIsTheDataStreamWriteIndex() {
         String dataStreamName = randomAlphaOfLength(10);
-        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
+        long ts = System.currentTimeMillis();
+        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts);
+        String failureIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts);
         String policyName = "test-ilm-policy";
         IndexMetadata sourceIndexMetadata = IndexMetadata.builder(indexName)
             .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
+        IndexMetadata failureSourceIndexMetadata = IndexMetadata.builder(failureIndexName)
+            .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(emptyClusterState())
             .metadata(
                 Metadata.builder()
                     .put(sourceIndexMetadata, true)
-                    .put(newInstance(dataStreamName, List.of(sourceIndexMetadata.getIndex())))
+                    .put(failureSourceIndexMetadata, true)
+                    .put(
+                        newInstance(dataStreamName, List.of(sourceIndexMetadata.getIndex()), List.of(failureSourceIndexMetadata.getIndex()))
+                    )
                     .build()
             )
             .build();
 
-        expectThrows(IllegalStateException.class, () -> createRandomInstance().performAction(sourceIndexMetadata.getIndex(), clusterState));
+        boolean useFailureStore = randomBoolean();
+        IndexMetadata indexToOperateOn = useFailureStore ? failureSourceIndexMetadata : sourceIndexMetadata;
+        expectThrows(IllegalStateException.class, () -> createRandomInstance().performAction(indexToOperateOn.getIndex(), clusterState));
     }
 
     public void testPerformActionThrowsExceptionIfTargetIndexIsMissing() {
         String dataStreamName = randomAlphaOfLength(10);
-        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
+        long ts = System.currentTimeMillis();
+        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts);
+        String failureIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts);
         String policyName = "test-ilm-policy";
         IndexMetadata sourceIndexMetadata = IndexMetadata.builder(indexName)
             .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
+        IndexMetadata failureSourceIndexMetadata = IndexMetadata.builder(failureIndexName)
+            .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
 
-        String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2);
+        String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2, ts);
+        String failureWriteIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 2, ts);
         IndexMetadata writeIndexMetadata = IndexMetadata.builder(writeIndexName)
             .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
+        IndexMetadata failureWriteIndexMetadata = IndexMetadata.builder(failureWriteIndexName)
+            .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
 
         List<Index> backingIndices = List.of(sourceIndexMetadata.getIndex(), writeIndexMetadata.getIndex());
+        List<Index> failureIndices = List.of(failureSourceIndexMetadata.getIndex(), failureWriteIndexMetadata.getIndex());
         ClusterState clusterState = ClusterState.builder(emptyClusterState())
             .metadata(
                 Metadata.builder()
                     .put(sourceIndexMetadata, true)
                     .put(writeIndexMetadata, true)
-                    .put(newInstance(dataStreamName, backingIndices))
+                    .put(failureSourceIndexMetadata, true)
+                    .put(failureWriteIndexMetadata, true)
+                    .put(newInstance(dataStreamName, backingIndices, failureIndices))
                     .build()
             )
             .build();
 
-        expectThrows(IllegalStateException.class, () -> createRandomInstance().performAction(sourceIndexMetadata.getIndex(), clusterState));
+        boolean useFailureStore = randomBoolean();
+        IndexMetadata indexToOperateOn = useFailureStore ? failureSourceIndexMetadata : sourceIndexMetadata;
+        expectThrows(IllegalStateException.class, () -> createRandomInstance().performAction(indexToOperateOn.getIndex(), clusterState));
     }
 
     public void testPerformActionIsNoOpIfIndexIsMissing() {
@@ -129,23 +159,39 @@ public void testPerformActionIsNoOpIfIndexIsMissing() {
 
     public void testPerformAction() {
         String dataStreamName = randomAlphaOfLength(10);
-        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
+        long ts = System.currentTimeMillis();
+        String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts);
+        String failureIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts);
         String policyName = "test-ilm-policy";
         IndexMetadata sourceIndexMetadata = IndexMetadata.builder(indexName)
             .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
+        IndexMetadata failureSourceIndexMetadata = IndexMetadata.builder(failureIndexName)
+            .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
 
-        String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2);
+        String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2, ts);
+        String failureWriteIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 2, ts);
         IndexMetadata writeIndexMetadata = IndexMetadata.builder(writeIndexName)
             .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
+        IndexMetadata failureWriteIndexMetadata = IndexMetadata.builder(failureWriteIndexName)
+            .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
+
+        boolean useFailureStore = randomBoolean();
+        String indexNameToUse = useFailureStore ? failureIndexName : indexName;
 
         String indexPrefix = "test-prefix-";
-        String targetIndex = indexPrefix + indexName;
+        String targetIndex = indexPrefix + indexNameToUse;
 
         IndexMetadata targetIndexMetadata = IndexMetadata.builder(targetIndex)
             .settings(settings(IndexVersion.current()))
@@ -154,12 +200,15 @@ public void testPerformAction() {
             .build();
 
         List<Index> backingIndices = List.of(sourceIndexMetadata.getIndex(), writeIndexMetadata.getIndex());
+        List<Index> failureIndices = List.of(failureSourceIndexMetadata.getIndex(), failureWriteIndexMetadata.getIndex());
         ClusterState clusterState = ClusterState.builder(emptyClusterState())
             .metadata(
                 Metadata.builder()
                     .put(sourceIndexMetadata, true)
                     .put(writeIndexMetadata, true)
-                    .put(newInstance(dataStreamName, backingIndices))
+                    .put(failureSourceIndexMetadata, true)
+                    .put(failureWriteIndexMetadata, true)
+                    .put(newInstance(dataStreamName, backingIndices, failureIndices))
                     .put(targetIndexMetadata, true)
                     .build()
             )
@@ -168,12 +217,16 @@ public void testPerformAction() {
         ReplaceDataStreamBackingIndexStep replaceSourceIndexStep = new ReplaceDataStreamBackingIndexStep(
             randomStepKey(),
             randomStepKey(),
-            (index, state) -> indexPrefix + index
+            (index, state) -> indexPrefix + indexNameToUse
         );
-        ClusterState newState = replaceSourceIndexStep.performAction(sourceIndexMetadata.getIndex(), clusterState);
+        IndexMetadata indexToOperateOn = useFailureStore ? failureSourceIndexMetadata : sourceIndexMetadata;
+        ClusterState newState = replaceSourceIndexStep.performAction(indexToOperateOn.getIndex(), clusterState);
         DataStream updatedDataStream = newState.metadata().dataStreams().get(dataStreamName);
-        assertThat(updatedDataStream.getIndices().size(), is(2));
-        assertThat(updatedDataStream.getIndices().get(0), is(targetIndexMetadata.getIndex()));
+        DataStream.DataStreamIndices resultIndices = useFailureStore
+            ? updatedDataStream.getFailureIndices()
+            : updatedDataStream.getBackingIndices();
+        assertThat(resultIndices.getIndices().size(), is(2));
+        assertThat(resultIndices.getIndices().get(0), is(targetIndexMetadata.getIndex()));
     }
 
     /**
@@ -181,23 +234,38 @@ public void testPerformAction() {
      */
     public void testPerformActionSameOriginalTargetError() {
         String dataStreamName = randomAlphaOfLength(10);
-        String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2);
+        long ts = System.currentTimeMillis();
+        String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2, ts);
+        String failureWriteIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 2, ts);
         String indexName = writeIndexName;
+        String failureIndexName = failureWriteIndexName;
         String policyName = "test-ilm-policy";
         IndexMetadata sourceIndexMetadata = IndexMetadata.builder(indexName)
             .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
+        IndexMetadata failureSourceIndexMetadata = IndexMetadata.builder(failureIndexName)
+            .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
 
         IndexMetadata writeIndexMetadata = IndexMetadata.builder(writeIndexName)
             .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
+        IndexMetadata failureWriteIndexMetadata = IndexMetadata.builder(failureWriteIndexName)
+            .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
 
         String indexPrefix = "test-prefix-";
-        String targetIndex = indexPrefix + indexName;
+        boolean useFailureStore = randomBoolean();
+        String indexNameToUse = useFailureStore ? failureIndexName : indexName;
+        String targetIndex = indexPrefix + indexNameToUse;
 
         IndexMetadata targetIndexMetadata = IndexMetadata.builder(targetIndex)
             .settings(settings(IndexVersion.current()))
@@ -206,12 +274,15 @@ public void testPerformActionSameOriginalTargetError() {
             .build();
 
         List<Index> backingIndices = List.of(writeIndexMetadata.getIndex());
+        List<Index> failureIndices = List.of(failureWriteIndexMetadata.getIndex());
         ClusterState clusterState = ClusterState.builder(emptyClusterState())
             .metadata(
                 Metadata.builder()
                     .put(sourceIndexMetadata, true)
                     .put(writeIndexMetadata, true)
-                    .put(newInstance(dataStreamName, backingIndices))
+                    .put(failureSourceIndexMetadata, true)
+                    .put(failureWriteIndexMetadata, true)
+                    .put(newInstance(dataStreamName, backingIndices, failureIndices))
                     .put(targetIndexMetadata, true)
                     .build()
             )
@@ -222,14 +293,17 @@ public void testPerformActionSameOriginalTargetError() {
             randomStepKey(),
             (index, state) -> indexPrefix + index
         );
+        IndexMetadata indexToOperateOn = useFailureStore ? failureSourceIndexMetadata : sourceIndexMetadata;
         IllegalStateException ex = expectThrows(
             IllegalStateException.class,
-            () -> replaceSourceIndexStep.performAction(sourceIndexMetadata.getIndex(), clusterState)
+            () -> replaceSourceIndexStep.performAction(indexToOperateOn.getIndex(), clusterState)
         );
         assertEquals(
             "index ["
-                + writeIndexName
-                + "] is the write index for data stream ["
+                + indexNameToUse
+                + "] is the "
+                + (useFailureStore ? "failure store " : "")
+                + "write index for data stream ["
                 + dataStreamName
                 + "], pausing ILM execution of lifecycle [test-ilm-policy] until this index is no longer the write index for the data "
                 + "stream via manual or automated rollover",
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java
index 1fcfc1fb287c4..f25a862362540 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java
@@ -96,7 +96,13 @@ public void testPerformAction() throws Exception {
 
     public void testPerformActionOnDataStream() throws Exception {
         String dataStreamName = "test-datastream";
-        IndexMetadata indexMetadata = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
+        long ts = System.currentTimeMillis();
+        IndexMetadata indexMetadata = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts))
+            .settings(settings(IndexVersion.current()))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
+        IndexMetadata failureIndexMetadata = IndexMetadata.builder(DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts))
             .settings(settings(IndexVersion.current()))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
@@ -107,9 +113,16 @@ public void testPerformActionOnDataStream() throws Exception {
         mockClientRolloverCall(dataStreamName);
 
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .metadata(Metadata.builder().put(newInstance(dataStreamName, List.of(indexMetadata.getIndex()))).put(indexMetadata, true))
+            .metadata(
+                Metadata.builder()
+                    .put(newInstance(dataStreamName, List.of(indexMetadata.getIndex()), List.of(failureIndexMetadata.getIndex())))
+                    .put(indexMetadata, true)
+                    .put(failureIndexMetadata, true)
+            )
             .build();
-        PlainActionFuture.<Void, Exception>get(f -> step.performAction(indexMetadata, clusterState, null, f));
+        boolean useFailureStore = randomBoolean();
+        IndexMetadata indexToOperateOn = useFailureStore ? failureIndexMetadata : indexMetadata;
+        PlainActionFuture.<Void, Exception>get(f -> step.performAction(indexToOperateOn, clusterState, null, f));
 
         Mockito.verify(client, Mockito.only()).admin();
         Mockito.verify(adminClient, Mockito.only()).indices();
@@ -118,13 +131,24 @@ public void testPerformActionOnDataStream() throws Exception {
 
     public void testSkipRolloverIfDataStreamIsAlreadyRolledOver() throws Exception {
         String dataStreamName = "test-datastream";
-        IndexMetadata firstGenerationIndex = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
+        long ts = System.currentTimeMillis();
+        IndexMetadata firstGenerationIndex = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts))
+            .settings(settings(IndexVersion.current()))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
+        IndexMetadata failureFirstGenerationIndex = IndexMetadata.builder(DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts))
             .settings(settings(IndexVersion.current()))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
 
-        IndexMetadata writeIndex = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 2))
+        IndexMetadata writeIndex = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 2, ts))
+            .settings(settings(IndexVersion.current()))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
+        IndexMetadata failureWriteIndex = IndexMetadata.builder(DataStream.getDefaultFailureStoreName(dataStreamName, 2, ts))
             .settings(settings(IndexVersion.current()))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
@@ -136,10 +160,20 @@ public void testSkipRolloverIfDataStreamIsAlreadyRolledOver() throws Exception {
                 Metadata.builder()
                     .put(firstGenerationIndex, true)
                     .put(writeIndex, true)
-                    .put(newInstance(dataStreamName, List.of(firstGenerationIndex.getIndex(), writeIndex.getIndex())))
+                    .put(failureFirstGenerationIndex, true)
+                    .put(failureWriteIndex, true)
+                    .put(
+                        newInstance(
+                            dataStreamName,
+                            List.of(firstGenerationIndex.getIndex(), writeIndex.getIndex()),
+                            List.of(failureFirstGenerationIndex.getIndex(), failureWriteIndex.getIndex())
+                        )
+                    )
             )
             .build();
-        PlainActionFuture.<Void, Exception>get(f -> step.performAction(firstGenerationIndex, clusterState, null, f));
+        boolean useFailureStore = randomBoolean();
+        IndexMetadata indexToOperateOn = useFailureStore ? failureFirstGenerationIndex : firstGenerationIndex;
+        PlainActionFuture.<Void, Exception>get(f -> step.performAction(indexToOperateOn, clusterState, null, f));
 
         verifyNoMoreInteractions(client);
         verifyNoMoreInteractions(adminClient);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java
index d9fd2d8a2247e..f5f36781e011b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java
@@ -170,13 +170,24 @@ public void testResultEvaluatedOnOnlyIndexTheAliasPointsToIfWriteIndexIsNull() {
 
     public void testResultEvaluatedOnDataStream() throws IOException {
         String dataStreamName = "test-datastream";
-        IndexMetadata originalIndexMeta = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
+        long ts = System.currentTimeMillis();
+        IndexMetadata originalIndexMeta = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts))
+            .settings(settings(IndexVersion.current()))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
+        IndexMetadata failureOriginalIndexMeta = IndexMetadata.builder(DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts))
             .settings(settings(IndexVersion.current()))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
 
-        IndexMetadata rolledIndexMeta = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 2))
+        IndexMetadata rolledIndexMeta = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 2, ts))
+            .settings(settings(IndexVersion.current()).put("index.write.wait_for_active_shards", "3"))
+            .numberOfShards(1)
+            .numberOfReplicas(3)
+            .build();
+        IndexMetadata failureRolledIndexMeta = IndexMetadata.builder(DataStream.getDefaultFailureStoreName(dataStreamName, 2, ts))
             .settings(settings(IndexVersion.current()).put("index.write.wait_for_active_shards", "3"))
             .numberOfShards(1)
             .numberOfReplicas(3)
@@ -186,28 +197,53 @@ public void testResultEvaluatedOnDataStream() throws IOException {
             ShardRoutingRoleStrategy.NO_SHARD_CREATION,
             rolledIndexMeta.getIndex()
         );
+        IndexRoutingTable.Builder failureRoutingTable = new IndexRoutingTable.Builder(
+            ShardRoutingRoleStrategy.NO_SHARD_CREATION,
+            failureRolledIndexMeta.getIndex()
+        );
         routingTable.addShard(
             TestShardRouting.newShardRouting(rolledIndexMeta.getIndex().getName(), 0, "node", null, true, ShardRoutingState.STARTED)
         );
         routingTable.addShard(
             TestShardRouting.newShardRouting(rolledIndexMeta.getIndex().getName(), 0, "node2", null, false, ShardRoutingState.STARTED)
         );
+        failureRoutingTable.addShard(
+            TestShardRouting.newShardRouting(failureRolledIndexMeta.getIndex().getName(), 0, "node", null, true, ShardRoutingState.STARTED)
+        );
+        failureRoutingTable.addShard(
+            TestShardRouting.newShardRouting(
+                failureRolledIndexMeta.getIndex().getName(),
+                0,
+                "node2",
+                null,
+                false,
+                ShardRoutingState.STARTED
+            )
+        );
 
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
             .metadata(
                 Metadata.builder()
                     .put(
-                        DataStreamTestHelper.newInstance(dataStreamName, List.of(originalIndexMeta.getIndex(), rolledIndexMeta.getIndex()))
+                        DataStreamTestHelper.newInstance(
+                            dataStreamName,
+                            List.of(originalIndexMeta.getIndex(), rolledIndexMeta.getIndex()),
+                            List.of(failureOriginalIndexMeta.getIndex(), failureRolledIndexMeta.getIndex())
+                        )
                     )
                     .put(originalIndexMeta, true)
                     .put(rolledIndexMeta, true)
+                    .put(failureOriginalIndexMeta, true)
+                    .put(failureRolledIndexMeta, true)
             )
-            .routingTable(RoutingTable.builder().add(routingTable.build()).build())
+            .routingTable(RoutingTable.builder().add(routingTable.build()).add(failureRoutingTable.build()).build())
             .build();
 
         WaitForActiveShardsStep waitForActiveShardsStep = createRandomInstance();
 
-        ClusterStateWaitStep.Result result = waitForActiveShardsStep.isConditionMet(originalIndexMeta.getIndex(), clusterState);
+        boolean useFailureStore = randomBoolean();
+        IndexMetadata indexToOperateOn = useFailureStore ? failureOriginalIndexMeta : originalIndexMeta;
+        ClusterStateWaitStep.Result result = waitForActiveShardsStep.isConditionMet(indexToOperateOn.getIndex(), clusterState);
         assertThat(result.isComplete(), is(false));
 
         XContentBuilder expected = new WaitForActiveShardsStep.ActiveShardsInfo(2, "3", false).toXContent(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java
index 2d39d093d149e..15958e9396d81 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContentObject;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import java.util.Collections;
@@ -47,6 +48,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -254,7 +256,14 @@ public void onFailure(Exception e) {
 
     public void testEvaluateConditionOnDataStreamTarget() {
         String dataStreamName = "test-datastream";
-        IndexMetadata indexMetadata = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
+        long ts = System.currentTimeMillis();
+        boolean failureStoreIndex = randomBoolean();
+        IndexMetadata indexMetadata = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts))
+            .settings(settings(IndexVersion.current()))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
+        IndexMetadata failureStoreMetadata = IndexMetadata.builder(DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts))
             .settings(settings(IndexVersion.current()))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
@@ -267,9 +276,17 @@ public void testEvaluateConditionOnDataStreamTarget() {
         SetOnce<Boolean> conditionsMet = new SetOnce<>();
         Metadata metadata = Metadata.builder()
             .put(indexMetadata, true)
-            .put(DataStreamTestHelper.newInstance(dataStreamName, List.of(indexMetadata.getIndex())))
+            .put(failureStoreMetadata, true)
+            .put(
+                DataStreamTestHelper.newInstance(
+                    dataStreamName,
+                    List.of(indexMetadata.getIndex()),
+                    List.of(failureStoreMetadata.getIndex())
+                )
+            )
             .build();
-        step.evaluateCondition(metadata, indexMetadata.getIndex(), new AsyncWaitStep.Listener() {
+        IndexMetadata indexToOperateOn = failureStoreIndex ? failureStoreMetadata : indexMetadata;
+        step.evaluateCondition(metadata, indexToOperateOn.getIndex(), new AsyncWaitStep.Listener() {
 
             @Override
             public void onResponse(boolean complete, ToXContentObject infomationContext) {
@@ -286,18 +303,38 @@ public void onFailure(Exception e) {
 
         verify(client, Mockito.only()).admin();
         verify(adminClient, Mockito.only()).indices();
-        verify(indicesClient, Mockito.only()).rolloverIndex(Mockito.any(), Mockito.any());
+
+        ArgumentCaptor<RolloverRequest> requestCaptor = ArgumentCaptor.forClass(RolloverRequest.class);
+        verify(indicesClient, Mockito.only()).rolloverIndex(requestCaptor.capture(), Mockito.any());
+
+        RolloverRequest request = requestCaptor.getValue();
+        assertThat(request.indicesOptions().failureStoreOptions().includeFailureIndices(), equalTo(failureStoreIndex));
+        assertThat(request.indicesOptions().failureStoreOptions().includeRegularIndices(), not(equalTo(failureStoreIndex)));
     }
 
     public void testSkipRolloverIfDataStreamIsAlreadyRolledOver() {
         String dataStreamName = "test-datastream";
-        IndexMetadata firstGenerationIndex = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
+        long ts = System.currentTimeMillis();
+        boolean failureStoreIndex = randomBoolean();
+        IndexMetadata firstGenerationIndex = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1, ts))
             .settings(settings(IndexVersion.current()))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
             .build();
 
-        IndexMetadata writeIndex = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 2))
+        IndexMetadata writeIndex = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 2, ts))
+            .settings(settings(IndexVersion.current()))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
+
+        IndexMetadata firstGenerationFailureIndex = IndexMetadata.builder(DataStream.getDefaultFailureStoreName(dataStreamName, 1, ts))
+            .settings(settings(IndexVersion.current()))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
+
+        IndexMetadata writeFailureIndex = IndexMetadata.builder(DataStream.getDefaultFailureStoreName(dataStreamName, 2, ts))
             .settings(settings(IndexVersion.current()))
             .numberOfShards(randomIntBetween(1, 5))
             .numberOfReplicas(randomIntBetween(0, 5))
@@ -308,9 +345,18 @@ public void testSkipRolloverIfDataStreamIsAlreadyRolledOver() {
         Metadata metadata = Metadata.builder()
             .put(firstGenerationIndex, true)
             .put(writeIndex, true)
-            .put(DataStreamTestHelper.newInstance(dataStreamName, List.of(firstGenerationIndex.getIndex(), writeIndex.getIndex())))
+            .put(firstGenerationFailureIndex, true)
+            .put(writeFailureIndex, true)
+            .put(
+                DataStreamTestHelper.newInstance(
+                    dataStreamName,
+                    List.of(firstGenerationIndex.getIndex(), writeIndex.getIndex()),
+                    List.of(firstGenerationFailureIndex.getIndex(), writeFailureIndex.getIndex())
+                )
+            )
             .build();
-        step.evaluateCondition(metadata, firstGenerationIndex.getIndex(), new AsyncWaitStep.Listener() {
+        IndexMetadata indexToOperateOn = failureStoreIndex ? firstGenerationFailureIndex : firstGenerationIndex;
+        step.evaluateCondition(metadata, indexToOperateOn.getIndex(), new AsyncWaitStep.Listener() {
 
             @Override
             public void onResponse(boolean complete, ToXContentObject infomationContext) {
@@ -665,7 +711,7 @@ public void testCreateRolloverRequestRolloverOnlyIfHasDocuments() {
         String rolloverTarget = randomAlphaOfLength(5);
         TimeValue masterTimeout = randomPositiveTimeValue();
 
-        RolloverRequest request = step.createRolloverRequest(rolloverTarget, masterTimeout, rolloverOnlyIfHasDocuments);
+        RolloverRequest request = step.createRolloverRequest(rolloverTarget, masterTimeout, rolloverOnlyIfHasDocuments, false);
 
         assertThat(request.getRolloverTarget(), is(rolloverTarget));
         assertThat(request.masterNodeTimeout(), is(masterTimeout));
@@ -704,7 +750,7 @@ public void testCreateRolloverRequestRolloverBeyondMaximumPrimaryShardDocCount()
             c.getMinDocs(),
             c.getMinPrimaryShardDocs()
         );
-        RolloverRequest request = step.createRolloverRequest(rolloverTarget, masterTimeout, true);
+        RolloverRequest request = step.createRolloverRequest(rolloverTarget, masterTimeout, true, false);
         assertThat(request.getRolloverTarget(), is(rolloverTarget));
         assertThat(request.masterNodeTimeout(), is(masterTimeout));
         assertThat(request.isDryRun(), is(true)); // it's always a dry_run
@@ -725,7 +771,7 @@ public void testCreateRolloverRequestRolloverBeyondMaximumPrimaryShardDocCount()
             c.getMinDocs(),
             c.getMinPrimaryShardDocs()
         );
-        request = step.createRolloverRequest(rolloverTarget, masterTimeout, true);
+        request = step.createRolloverRequest(rolloverTarget, masterTimeout, true, false);
         assertThat(request.getRolloverTarget(), is(rolloverTarget));
         assertThat(request.masterNodeTimeout(), is(masterTimeout));
         assertThat(request.isDryRun(), is(true)); // it's always a dry_run
@@ -747,7 +793,7 @@ public void testCreateRolloverRequestRolloverBeyondMaximumPrimaryShardDocCount()
             c.getMinDocs(),
             c.getMinPrimaryShardDocs()
         );
-        request = step.createRolloverRequest(rolloverTarget, masterTimeout, true);
+        request = step.createRolloverRequest(rolloverTarget, masterTimeout, true, false);
         assertThat(request.getRolloverTarget(), is(rolloverTarget));
         assertThat(request.masterNodeTimeout(), is(masterTimeout));
         assertThat(request.isDryRun(), is(true)); // it's always a dry_run

From 1558bb8f54f6054bf9a12151094a1bafb43e9153 Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Wed, 5 Jun 2024 17:02:35 -0400
Subject: [PATCH 183/208] Remove unused core expressions (#109406)

Removes the leaf expressions for arithmetic operations from esql-core.
These aren't used in ESQL, and we don't want to refer to them by
accident.

Earlier work in https://github.com/elastic/elasticsearch/pull/109216
migrated the remaining references to these classes to refer to their
esql versions.
---
 .../predicate/operator/arithmetic/Add.java    | 45 -------------
 .../DateTimeArithmeticOperation.java          | 39 -----------
 .../predicate/operator/arithmetic/Div.java    | 53 ---------------
 .../predicate/operator/arithmetic/Mod.java    | 34 ----------
 .../predicate/operator/arithmetic/Mul.java    | 66 -------------------
 .../predicate/operator/arithmetic/Sub.java    | 36 ----------
 6 files changed, 273 deletions(-)
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Add.java
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Div.java
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mod.java
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mul.java
 delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Sub.java

diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Add.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Add.java
deleted file mode 100644
index 5b16b478f6519..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Add.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic;
-
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-
-/**
- * Addition function ({@code a + b}).
- */
-public class Add extends DateTimeArithmeticOperation implements BinaryComparisonInversible {
-    public Add(Source source, Expression left, Expression right) {
-        super(source, left, right, DefaultBinaryArithmeticOperation.ADD);
-    }
-
-    @Override
-    protected NodeInfo<Add> info() {
-        return NodeInfo.create(this, Add::new, left(), right());
-    }
-
-    @Override
-    protected Add replaceChildren(Expression left, Expression right) {
-        return new Add(source(), left, right);
-    }
-
-    @Override
-    public Add swapLeftAndRight() {
-        return new Add(source(), right(), left());
-    }
-
-    @Override
-    public ArithmeticOperationFactory binaryComparisonInverse() {
-        return Sub::new;
-    }
-
-    @Override
-    protected boolean isCommutative() {
-        return true;
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
deleted file mode 100644
index 9e08cea749a34..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic;
-
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-
-abstract class DateTimeArithmeticOperation extends ArithmeticOperation {
-
-    DateTimeArithmeticOperation(Source source, Expression left, Expression right, BinaryArithmeticOperation operation) {
-        super(source, left, right, operation);
-    }
-
-    @Override
-    protected TypeResolution resolveType() {
-        if (childrenResolved() == false) {
-            return new TypeResolution("Unresolved children");
-        }
-
-        // arithmetic operation can work on numbers in QL
-
-        DataType l = left().dataType();
-        DataType r = right().dataType();
-
-        // 1. both are numbers
-        if (l.isNumeric() && r.isNumeric()) {
-            return TypeResolution.TYPE_RESOLVED;
-        }
-
-        // fall-back to default checks
-        return super.resolveType();
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Div.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Div.java
deleted file mode 100644
index 5f4c660479579..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Div.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic;
-
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.core.type.DataTypeConverter;
-
-/**
- * Division function ({@code a / b}).
- */
-public class Div extends ArithmeticOperation implements BinaryComparisonInversible {
-
-    private DataType dataType;
-
-    public Div(Source source, Expression left, Expression right) {
-        this(source, left, right, null);
-    }
-
-    public Div(Source source, Expression left, Expression right, DataType dataType) {
-        super(source, left, right, DefaultBinaryArithmeticOperation.DIV);
-        this.dataType = dataType;
-    }
-
-    @Override
-    protected NodeInfo<Div> info() {
-        return NodeInfo.create(this, Div::new, left(), right(), dataType);
-    }
-
-    @Override
-    protected Div replaceChildren(Expression newLeft, Expression newRight) {
-        return new Div(source(), newLeft, newRight, dataType);
-    }
-
-    @Override
-    public DataType dataType() {
-        if (dataType == null) {
-            dataType = DataTypeConverter.commonType(left().dataType(), right().dataType());
-        }
-        return dataType;
-    }
-
-    @Override
-    public ArithmeticOperationFactory binaryComparisonInverse() {
-        return Mul::new;
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mod.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mod.java
deleted file mode 100644
index dea7d4e02e0b3..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mod.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic;
-
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-
-/**
- * <a href="https://en.wikipedia.org/wiki/Modulo_operation">Modulo</a>
- * function ({@code a % b}).
- *
- * Note this operator is also registered as a function (needed for ODBC/SQL) purposes.
- */
-public class Mod extends ArithmeticOperation {
-
-    public Mod(Source source, Expression left, Expression right) {
-        super(source, left, right, DefaultBinaryArithmeticOperation.MOD);
-    }
-
-    @Override
-    protected NodeInfo<Mod> info() {
-        return NodeInfo.create(this, Mod::new, left(), right());
-    }
-
-    @Override
-    protected Mod replaceChildren(Expression newLeft, Expression newRight) {
-        return new Mod(source(), newLeft, newRight);
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mul.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mul.java
deleted file mode 100644
index db46ecf81ea1d..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Mul.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic;
-
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-
-import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
-
-/**
- * Multiplication function ({@code a * b}).
- */
-public class Mul extends ArithmeticOperation implements BinaryComparisonInversible {
-
-    public Mul(Source source, Expression left, Expression right) {
-        super(source, left, right, DefaultBinaryArithmeticOperation.MUL);
-    }
-
-    @Override
-    protected TypeResolution resolveType() {
-        if (childrenResolved() == false) {
-            return new TypeResolution("Unresolved children");
-        }
-
-        DataType l = left().dataType();
-        DataType r = right().dataType();
-
-        // 1. both are numbers
-        if (DataType.isNullOrNumeric(l) && DataType.isNullOrNumeric(r)) {
-            return TypeResolution.TYPE_RESOLVED;
-        }
-
-        return new TypeResolution(format(null, "[{}] has arguments with incompatible types [{}] and [{}]", symbol(), l, r));
-    }
-
-    @Override
-    protected NodeInfo<Mul> info() {
-        return NodeInfo.create(this, Mul::new, left(), right());
-    }
-
-    @Override
-    protected Mul replaceChildren(Expression newLeft, Expression newRight) {
-        return new Mul(source(), newLeft, newRight);
-    }
-
-    @Override
-    public Mul swapLeftAndRight() {
-        return new Mul(source(), right(), left());
-    }
-
-    @Override
-    public ArithmeticOperationFactory binaryComparisonInverse() {
-        return Div::new;
-    }
-
-    @Override
-    protected boolean isCommutative() {
-        return true;
-    }
-}
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Sub.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Sub.java
deleted file mode 100644
index 8a345986e5fba..0000000000000
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Sub.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic;
-
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-
-/**
- * Subtraction function ({@code a - b}).
- */
-public class Sub extends DateTimeArithmeticOperation implements BinaryComparisonInversible {
-
-    public Sub(Source source, Expression left, Expression right) {
-        super(source, left, right, DefaultBinaryArithmeticOperation.SUB);
-    }
-
-    @Override
-    protected NodeInfo<Sub> info() {
-        return NodeInfo.create(this, Sub::new, left(), right());
-    }
-
-    @Override
-    protected Sub replaceChildren(Expression newLeft, Expression newRight) {
-        return new Sub(source(), newLeft, newRight);
-    }
-
-    @Override
-    public ArithmeticOperationFactory binaryComparisonInverse() {
-        return Add::new;
-    }
-}

From 5af24ae221069518fe72ac880edec741f91ebc78 Mon Sep 17 00:00:00 2001
From: elasticsearchmachine
 <58790826+elasticsearchmachine@users.noreply.github.com>
Date: Thu, 6 Jun 2024 07:14:11 +1000
Subject: [PATCH 184/208] Mute
 org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppendTests
 testEvaluateBlockWithoutNulls {TestCase=<cartesian_shape>, <cartesian_shape>}
 #109409

---
 muted-tests.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 3d64f87144bd3..32128da9c3712 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -19,8 +19,7 @@ tests:
   method: "testGuessIsDayFirstFromLocale"
 - class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108857"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
-    \ dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
 - class: "org.elasticsearch.upgrades.SearchStatesIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108991"
   method: "testCanMatch"
@@ -29,8 +28,7 @@ tests:
   method: "testTrainedModelInference"
 - class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109188"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
-    \ dependent mappings / dates}"
+  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
 - class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109189"
   method: "test {p0=esql/70_locale/Date format with Italian locale}"
@@ -45,8 +43,7 @@ tests:
   method: "testTimestampFieldTypeExposedByAllIndicesServices"
 - class: "org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109318"
-  method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling\
-    \ (too complex pattern)}"
+  method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)}"
 - class: "org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT"
   issue: "https://github.com/elastic/elasticsearch/issues/101598"
   method: "testFeatureImportanceValues"
@@ -59,6 +56,9 @@ tests:
 - class: "org.elasticsearch.xpack.inference.InferenceCrudIT"
   issue: "https://github.com/elastic/elasticsearch/issues/109391"
   method: "testDeleteEndpointWhileReferencedByPipeline"
+- class: org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppendTests
+  method: testEvaluateBlockWithoutNulls {TestCase=<cartesian_shape>, <cartesian_shape>}
+  issue: https://github.com/elastic/elasticsearch/issues/109409
 
 # Examples:
 #

From 703c7ad5c61f435be6cc2c026e69d5add72a254d Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Wed, 5 Jun 2024 17:22:50 -0400
Subject: [PATCH 185/208] [ESQL] Rough index of key classes and packages
 (#109405)

First draft of some top-level developer docs for ESQL.  At the moment
this is just a collection of links to key classes and packages, but we
can add to it as we find useful.
---
 .../xpack/esql/package-info.java              | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java

diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java
new file mode 100644
index 0000000000000..863476ba55686
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * ES|QL Overview and Documentation Links
+ *
+ * <h2>Major Components</h2>
+ * <ul>
+ * <li>{@link org.elasticsearch.compute} - The compute engine drives query execution
+ * <ul>
+ *     <li>{@link org.elasticsearch.compute.data.Block} - fundamental unit of data.  Operations vectorize over blocks.</li>
+ *     <li>{@link org.elasticsearch.compute.data.Page} - Data is broken up into pages (which are collections of blocks) to
+ *     manage size in memory</li>
+ * </ul>
+ * </li>
+ * <li>{@link org.elasticsearch.xpack.esql.core} - Core Utility Classes
+ * <ul>
+ *     <li>{@link org.elasticsearch.xpack.esql.core.type.DataType} - ES|QL is a typed language, and all the supported data types
+ *     are listed in this collection.</li>
+ *     <li>{@link org.elasticsearch.xpack.esql.core.expression.Expression} - Expression is the basis for all functions in ES|QL,
+ *     but see also {@link org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper}</li>
+ * </ul>
+ * </li>
+ * <li> org.elasticsearch.compute.gen - ES|QL generates code for evaluators, which are type-specific implementations of
+ * functions, designed to run over a {@link org.elasticsearch.compute.data.Block} </li>
+ * <li>{@link org.elasticsearch.xpack.esql.session.EsqlSession} - manages state across a query</li>
+ * <li>{@link org.elasticsearch.xpack.esql.expression.function.scalar} - Guide to writing scalar functions</li>
+ * <li>{@link org.elasticsearch.xpack.esql.analysis.Analyzer} - The first step in query processing</li>
+ * <li>{@link org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer} - Coordinator level logical optimizations</li>
+ * <li>{@link org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer} - Data node level logical optimizations</li>
+ * <li>{@link org.elasticsearch.xpack.esql.action.RestEsqlQueryAction} - REST API entry point</li>
+ * </ul>
+ */
+
+package org.elasticsearch.xpack.esql;

From 84bc73bcfe07e2439030dd26806ff6f293d42954 Mon Sep 17 00:00:00 2001
From: Rene Groeschke <rene@elastic.co>
Date: Thu, 6 Jun 2024 07:28:58 +0200
Subject: [PATCH 186/208] More cc tweaks to esql build script (#109356)

---
 x-pack/plugin/esql/build.gradle | 34 +++++++++++++++++++++------------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle
index 669f38bd44ecc..faa5a118a90cd 100644
--- a/x-pack/plugin/esql/build.gradle
+++ b/x-pack/plugin/esql/build.gradle
@@ -47,24 +47,34 @@ tasks.named("compileJava").configure {
   exclude { it.file.toString().startsWith("${projectDir}/src/main/generated-src/generated") }
 }
 
+interface Injected {
+  @Inject FileSystemOperations getFs()
+}
+
 tasks.named("test").configure {
   if (BuildParams.isCi() == false) {
     systemProperty 'generateDocs', true
+    def injected = project.objects.newInstance(Injected)
     doFirst {
-      project.delete(
-        files("${projectDir}/build/testrun/test/temp/esql/functions")
-      )
+      injected.fs.delete {
+        it.delete("build/testrun/test/temp/esql/functions")
+      }
     }
+    File functionsFolder = file("build/testrun/test/temp/esql/functions")
+    File signatureFolder = file("build/testrun/test/temp/esql/functions/signature")
+    File typesFolder = file("build/testrun/test/temp/esql/functions/types")
+    def functionsDocFolder = file("${rootDir}/docs/reference/esql/functions")
+    def effectiveProjectDir = projectDir
+
     doLast {
-      List signatures = file("${projectDir}/build/testrun/test/temp/esql/functions/signature").list().findAll {it.endsWith("svg")}
-      List types = file("${projectDir}/build/testrun/test/temp/esql/functions/types").list().findAll {it.endsWith("asciidoc")}
+      List types = typesFolder.list().findAll {it.endsWith("asciidoc")}
       int count = types == null ? 0 : types.size()
       Closure readExample = line -> {
         line.replaceAll(/read-example::([^\[]+)\[tag=([^,\]]+)(, ?json)?\]/, {
           String file = it[1]
           String tag = it[2]
           boolean isJson = it[3]
-          String allExamples = new File("${projectDir}/qa/testFixtures/src/main/resources/${file}").text
+          String allExamples = new File("${effectiveProjectDir}/qa/testFixtures/src/main/resources/${file}").text
             .replaceAll(System.lineSeparator(), "\n")
           int start = allExamples.indexOf("tag::${tag}[]")
           int end = allExamples.indexOf("end::${tag}[]", start)
@@ -85,9 +95,9 @@ tasks.named("test").configure {
         logger.quiet("ESQL Docs: No function signatures created. Skipping sync.")
       } else if (count == 1) {
         logger.quiet("ESQL Docs: Only files related to $types, patching them into place")
-        project.sync {
-          from "${projectDir}/build/testrun/test/temp/esql/functions"
-          into "${rootDir}/docs/reference/esql/functions"
+        injected.fs.sync {
+          from functionsFolder
+          into functionsDocFolder
           include '**/*.asciidoc', '**/*.svg', '**/*.md', '**/*.json'
           preserve {
             include '/*.asciidoc', '**/*.asciidoc', '**/*.md', '**/*.json', '**/*.svg', 'README.md'
@@ -95,9 +105,9 @@ tasks.named("test").configure {
           filter readExample
         }
       } else {
-        project.sync {
-          from "${projectDir}/build/testrun/test/temp/esql/functions"
-          into "${rootDir}/docs/reference/esql/functions"
+        injected.fs.sync {
+          from functionsFolder
+          into functionsDocFolder
           include '**/*.asciidoc', '**/*.svg', '**/*.md', '**/*.json'
           preserve {
             include '/*.asciidoc', 'README.md'

From fad8c191fc92d680279a9f415dd25e66edc2b86b Mon Sep 17 00:00:00 2001
From: Rene Groeschke <rene@elastic.co>
Date: Thu, 6 Jun 2024 07:30:21 +0200
Subject: [PATCH 187/208] Make BWC build logic configuration cache compatible
 (#109354)

More refactoring towards configuration cache compatibility
---
 .../main/groovy/elasticsearch.bwc-test.gradle |  7 +-
 .../gradle/internal/BwcSetupExtension.java    | 72 ++++++++++++++-----
 .../gradle/internal/InternalBwcGitPlugin.java |  8 +--
 .../InternalDistributionBwcSetupPlugin.java   | 42 +++++++++--
 .../org/elasticsearch/gradle/LoggedExec.java  |  6 +-
 5 files changed, 99 insertions(+), 36 deletions(-)

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle
index 5512b06d0ab8b..ff9b6fe7a526d 100644
--- a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle
+++ b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle
@@ -33,7 +33,8 @@ tasks.register("bwcTest") {
 
 plugins.withType(ElasticsearchTestBasePlugin) {
   tasks.withType(Test).matching { it.name ==~ /v[0-9\.]+#.*/ }.configureEach {
-    onlyIf("BWC tests enabled") { project.bwc_tests_enabled }
+    boolean bwcEnabled = project.bwc_tests_enabled
+    onlyIf("BWC tests enabled") { bwcEnabled }
     nonInputProperties.systemProperty 'tests.bwc', 'true'
   }
 }
@@ -50,5 +51,5 @@ plugins.withType(InternalJavaRestTestPlugin) {
   }
 }
 
-tasks.matching { it.name.equals("check") }.configureEach {dependsOn(bwcTestSnapshots) }
-tasks.matching { it.name.equals("test") }.configureEach {enabled = false}
+tasks.matching { it.name.equals("check") }.configureEach { dependsOn(bwcTestSnapshots) }
+tasks.matching { it.name.equals("test") }.configureEach { enabled = false }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java
index 3d6d37575eca9..7010ed92d4c57 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java
@@ -20,6 +20,9 @@
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.provider.ValueSource;
+import org.gradle.api.provider.ValueSourceParameters;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainService;
@@ -41,6 +44,7 @@ public class BwcSetupExtension {
     private static final Version BUILD_TOOL_MINIMUM_VERSION = Version.fromString("7.14.0");
     private final Project project;
     private final ObjectFactory objectFactory;
+    private final ProviderFactory providerFactory;
     private final JavaToolchainService toolChainService;
     private final Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo;
 
@@ -49,12 +53,14 @@ public class BwcSetupExtension {
     public BwcSetupExtension(
         Project project,
         ObjectFactory objectFactory,
+        ProviderFactory providerFactory,
         JavaToolchainService toolChainService,
         Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo,
         Provider<File> checkoutDir
     ) {
         this.project = project;
         this.objectFactory = objectFactory;
+        this.providerFactory = providerFactory;
         this.toolChainService = toolChainService;
         this.unreleasedVersionInfo = unreleasedVersionInfo;
         this.checkoutDir = checkoutDir;
@@ -65,11 +71,26 @@ TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration)
     }
 
     TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration, boolean useUniqueUserHome) {
-        return createRunBwcGradleTask(project, name, configuration, useUniqueUserHome);
+        return createRunBwcGradleTask(
+            project,
+            checkoutDir,
+            providerFactory,
+            unreleasedVersionInfo,
+            objectFactory,
+            toolChainService,
+            name,
+            configuration,
+            useUniqueUserHome
+        );
     }
 
-    private TaskProvider<LoggedExec> createRunBwcGradleTask(
+    private static TaskProvider<LoggedExec> createRunBwcGradleTask(
         Project project,
+        Provider<File> checkoutDir,
+        ProviderFactory providerFactory,
+        Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo,
+        ObjectFactory objectFactory,
+        JavaToolchainService toolChainService,
         String name,
         Action<LoggedExec> configAction,
         boolean useUniqueUserHome
@@ -78,10 +99,10 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(
             loggedExec.dependsOn("checkoutBwcBranch");
             loggedExec.getWorkingDir().set(checkoutDir.get());
 
-            loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> {
-                String minimumCompilerVersion = readFromFile(new File(checkoutDir, minimumCompilerVersionPath(version.version())));
-                return getJavaHome(Integer.parseInt(minimumCompilerVersion));
-            }));
+            loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", providerFactory.of(JavaHomeValueSource.class, spec -> {
+                spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version()));
+                spec.getParameters().getCheckoutDir().set(checkoutDir);
+            }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s))));
 
             if (BuildParams.isCi() && OS.current() != OS.WINDOWS) {
                 // TODO: Disabled for now until we can figure out why files are getting corrupted
@@ -137,10 +158,13 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(
         });
     }
 
-    private String minimumCompilerVersionPath(Version bwcVersion) {
-        return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION))
-            ? "build-tools-internal/" + MINIMUM_COMPILER_VERSION_PATH
-            : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
+    /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
+    private static Provider<String> getJavaHome(ObjectFactory objectFactory, JavaToolchainService toolChainService, final int version) {
+        Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
+        return toolChainService.launcherFor(javaToolchainSpec -> {
+            javaToolchainSpec.getLanguageVersion().value(value);
+            javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE);
+        }).map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath());
     }
 
     private static String readFromFile(File file) {
@@ -151,13 +175,25 @@ private static String readFromFile(File file) {
         }
     }
 
-    /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
-    public String getJavaHome(final int version) {
-        Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
-        return toolChainService.launcherFor(javaToolchainSpec -> {
-            javaToolchainSpec.getLanguageVersion().value(value);
-            javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE);
-        }).get().getMetadata().getInstallationPath().getAsFile().getAbsolutePath();
-    }
+    public static abstract class JavaHomeValueSource implements ValueSource<String, JavaHomeValueSource.Params> {
+
+        private String minimumCompilerVersionPath(Version bwcVersion) {
+            return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION))
+                ? "build-tools-internal/" + MINIMUM_COMPILER_VERSION_PATH
+                : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
+        }
 
+        @Override
+        public String obtain() {
+            return readFromFile(
+                new File(getParameters().getCheckoutDir().get(), minimumCompilerVersionPath(getParameters().getVersion().get()))
+            );
+        }
+
+        public interface Params extends ValueSourceParameters {
+            Property<Version> getVersion();
+
+            Property<File> getCheckoutDir();
+        }
+    }
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java
index 71c76b2045007..7add1e615f577 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java
@@ -93,13 +93,6 @@ public void execute(Task task) {
                         String remoteRepo = remote.get();
                         // for testing only we can override the base remote url
                         String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo")
-                            .orElse(
-                                providerFactory.provider(
-                                    () -> addRemote.getExtensions().getExtraProperties().has("remote")
-                                        ? addRemote.getExtensions().getExtraProperties().get("remote").toString()
-                                        : null
-                                )
-                            )
                             .getOrElse("https://github.com/" + remoteRepo + "/" + rootProjectName);
                         spec.commandLine("git", "remote", "add", remoteRepo, remoteRepoUrl);
                     });
@@ -213,6 +206,7 @@ private String maybeAlignedRefSpec(Logger logger, String defaultRefSpec) {
 
     private void writeFile(File file, String content) {
         try {
+            file.getParentFile().mkdirs();
             Files.writeString(file.toPath(), content, CREATE, TRUNCATE_EXISTING);
         } catch (IOException e) {
             throw new UncheckedIOException(e);
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
index f727dc165a8a9..a2247adcf7b9e 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
@@ -16,6 +16,7 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
+import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.plugins.JvmToolchainsPlugin;
 import org.gradle.api.provider.Provider;
@@ -63,15 +64,39 @@ public void apply(Project project) {
         project.getPlugins().apply(JvmToolchainsPlugin.class);
         toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
         BuildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> {
-            configureBwcProject(project.project(unreleasedVersion.gradleProjectPath()), unreleasedVersion);
+            configureBwcProject(
+                project.project(unreleasedVersion.gradleProjectPath()),
+                unreleasedVersion,
+                providerFactory,
+                objectFactory,
+                toolChainService
+            );
         });
     }
 
-    private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionInfo versionInfo) {
+    private static void configureBwcProject(
+        Project project,
+        BwcVersions.UnreleasedVersionInfo versionInfo,
+        ProviderFactory providerFactory,
+        ObjectFactory objectFactory,
+        JavaToolchainService toolChainService
+    ) {
+        ProjectLayout layout = project.getLayout();
         Provider<BwcVersions.UnreleasedVersionInfo> versionInfoProvider = providerFactory.provider(() -> versionInfo);
-        Provider<File> checkoutDir = versionInfoProvider.map(info -> new File(project.getBuildDir(), "bwc/checkout-" + info.branch()));
+        Provider<File> checkoutDir = versionInfoProvider.map(
+            info -> new File(layout.getBuildDirectory().get().getAsFile(), "bwc/checkout-" + info.branch())
+        );
         BwcSetupExtension bwcSetupExtension = project.getExtensions()
-            .create("bwcSetup", BwcSetupExtension.class, project, objectFactory, toolChainService, versionInfoProvider, checkoutDir);
+            .create(
+                "bwcSetup",
+                BwcSetupExtension.class,
+                project,
+                objectFactory,
+                providerFactory,
+                toolChainService,
+                versionInfoProvider,
+                checkoutDir
+            );
         BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension();
         Provider<Version> bwcVersion = versionInfoProvider.map(info -> info.version());
         gitExtension.setBwcVersion(versionInfoProvider.map(info -> info.version()));
@@ -157,7 +182,7 @@ private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionI
         }
     }
 
-    private void registerBwcDistributionArtifacts(Project bwcProject, DistributionProject distributionProject) {
+    private static void registerBwcDistributionArtifacts(Project bwcProject, DistributionProject distributionProject) {
         String projectName = distributionProject.name;
         String buildBwcTask = buildBwcTaskName(projectName);
 
@@ -174,7 +199,11 @@ private void registerBwcDistributionArtifacts(Project bwcProject, DistributionPr
         }
     }
 
-    private void registerDistributionArchiveArtifact(Project bwcProject, DistributionProject distributionProject, String buildBwcTask) {
+    private static void registerDistributionArchiveArtifact(
+        Project bwcProject,
+        DistributionProject distributionProject,
+        String buildBwcTask
+    ) {
         File distFile = distributionProject.expectedBuildArtifact.distFile;
         String artifactFileName = distFile.getName();
         String artifactName = artifactFileName.contains("oss") ? "elasticsearch-oss" : "elasticsearch";
@@ -363,5 +392,4 @@ private static class DistributionProjectArtifact {
             this.expandedDistDir = expandedDistDir;
         }
     }
-
 }
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java
index 4fda91d332118..6087482db278d 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java
@@ -65,6 +65,9 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat
     @Optional
     abstract public MapProperty<String, String> getEnvironment();
 
+    @Internal
+    abstract public MapProperty<String, String> getNonTrackedEnvironment();
+
     @Input
     abstract public Property<String> getExecutable();
 
@@ -139,7 +142,8 @@ public void run() {
             execSpec.setStandardOutput(finalOutputStream);
             execSpec.setErrorOutput(finalOutputStream);
             execSpec.setExecutable(getExecutable().get());
-            execSpec.setEnvironment(getEnvironment().get());
+            execSpec.environment(getEnvironment().get());
+            execSpec.environment(getNonTrackedEnvironment().get());
             if (getArgs().isPresent()) {
                 execSpec.setArgs(getArgs().get());
             }

From 13afe0fda9c50a0adc5d17b2f1a994168ce6baf3 Mon Sep 17 00:00:00 2001
From: Mikhail Berezovskiy <mikhail@elastic.co>
Date: Wed, 5 Jun 2024 23:22:40 -0700
Subject: [PATCH 188/208] Change UnassignedInfo class to record (#109363)

---
 .../ClusterAllocationExplainIT.java           |  18 +-
 .../admin/indices/create/ShrinkIndexIT.java   |   4 +-
 .../cluster/routing/AllocationIdIT.java       |   2 +-
 .../cluster/routing/PrimaryAllocationIT.java  |   4 +-
 .../cluster/routing/ShardRoutingRoleIT.java   |   4 +-
 .../gateway/GatewayIndexStateIT.java          |   8 +-
 .../RemoveCorruptedShardDataCommandIT.java    |   4 +-
 .../index/store/CorruptedFileIT.java          |   2 +-
 .../index/store/CorruptedTranslogIT.java      |   4 +-
 .../indices/cluster/ShardLockFailureIT.java   |   2 +-
 .../SharedClusterSnapshotRestoreIT.java       |   8 +-
 .../ClusterAllocationExplanation.java         |  12 +-
 .../cluster/health/ClusterShardHealth.java    |   4 +-
 .../cluster/routing/IndexRoutingTable.java    |  18 +-
 .../cluster/routing/RoutingNodes.java         |  48 ++--
 .../cluster/routing/ShardRouting.java         |   4 +-
 .../cluster/routing/UnassignedInfo.java       | 207 ++++++------------
 .../routing/allocation/AllocationService.java |  28 +--
 .../allocation/ShardChangesObserver.java      |   2 +-
 .../allocator/BalancedShardsAllocator.java    |  20 +-
 .../allocator/DesiredBalanceComputer.java     |  35 ++-
 .../allocator/DesiredBalanceReconciler.java   |  20 +-
 ...AllocateEmptyPrimaryAllocationCommand.java |  10 +-
 .../decider/MaxRetryAllocationDecider.java    |   2 +-
 .../RestoreInProgressAllocationDecider.java   |   2 +-
 ...rdsAvailabilityHealthIndicatorService.java |  10 +-
 .../gateway/ReplicaShardAllocator.java        |  12 +-
 .../rest/action/cat/RestShardsAction.java     |   8 +-
 .../snapshots/RestoreService.java             |   6 +-
 .../ClusterAllocationExplainActionTests.java  |   6 +-
 .../cluster/reroute/ClusterRerouteTests.java  |   2 +-
 .../health/ClusterStateHealthTests.java       |   4 +-
 .../MetadataIndexStateServiceTests.java       |   2 +-
 .../DelayedAllocationServiceTests.java        |  10 +-
 .../cluster/routing/ShardRoutingTests.java    |   2 +-
 .../cluster/routing/UnassignedInfoTests.java  |  78 +++----
 .../MaxRetryAllocationDeciderTests.java       |  36 +--
 .../TrackFailedAllocationNodesTests.java      |   9 +-
 .../DesiredBalanceComputerTests.java          |  22 +-
 .../DesiredBalanceReconcilerTests.java        |  30 +--
 .../DesiredBalanceShardsAllocatorTests.java   |   6 +-
 .../decider/DiskThresholdDeciderTests.java    |   6 +-
 ...storeInProgressAllocationDeciderTests.java |  18 +-
 .../gateway/PrimaryShardAllocatorTests.java   |   6 +-
 .../gateway/ReplicaShardAllocatorTests.java   |   4 +-
 .../snapshots/SnapshotResiliencyTests.java    |   2 +-
 .../cluster/ESAllocationTestCase.java         |   4 +-
 .../xpack/ccr/CcrRepositoryIT.java            |   4 +-
 .../ClusterStateApplierOrderingTests.java     |   2 +-
 .../SearchableSnapshotAllocator.java          |   4 +-
 .../TransportGetShutdownStatusAction.java     |   2 +-
 51 files changed, 348 insertions(+), 419 deletions(-)

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
index 26b33acfcbe98..897f10b031dcb 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
@@ -100,10 +100,10 @@ public void testUnassignedPrimaryWithExistingIndex() throws Exception {
 
         // verify unassigned info
         assertNotNull(unassignedInfo);
-        assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
+        assertEquals(Reason.NODE_LEFT, unassignedInfo.reason());
         assertTrue(
-            unassignedInfo.getLastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA
-                || unassignedInfo.getLastAllocationStatus() == AllocationStatus.NO_VALID_SHARD_COPY
+            unassignedInfo.lastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA
+                || unassignedInfo.lastAllocationStatus() == AllocationStatus.NO_VALID_SHARD_COPY
         );
 
         // verify cluster info
@@ -190,8 +190,8 @@ public void testUnassignedReplicaDelayedAllocation() throws Exception {
 
         // verify unassigned info
         assertNotNull(unassignedInfo);
-        assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
-        assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.getLastAllocationStatus());
+        assertEquals(Reason.NODE_LEFT, unassignedInfo.reason());
+        assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.lastAllocationStatus());
 
         // verify cluster info
         verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
@@ -320,8 +320,8 @@ public void testUnassignedReplicaWithPriorCopy() throws Exception {
 
         // verify unassigned info
         assertNotNull(unassignedInfo);
-        assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
-        assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.getLastAllocationStatus());
+        assertEquals(Reason.NODE_LEFT, unassignedInfo.reason());
+        assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.lastAllocationStatus());
 
         // verify cluster info
         verifyClusterInfo(clusterInfo, includeDiskInfo, 3);
@@ -432,8 +432,8 @@ public void testAllocationFilteringOnIndexCreation() throws Exception {
 
         // verify unassigned info
         assertNotNull(unassignedInfo);
-        assertEquals(Reason.INDEX_CREATED, unassignedInfo.getReason());
-        assertEquals(AllocationStatus.DECIDERS_NO, unassignedInfo.getLastAllocationStatus());
+        assertEquals(Reason.INDEX_CREATED, unassignedInfo.reason());
+        assertEquals(AllocationStatus.DECIDERS_NO, unassignedInfo.lastAllocationStatus());
 
         // verify cluster info
         verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
index aa4fee3a3f94d..61e5c1bfcc811 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
@@ -386,9 +386,9 @@ public void testCreateShrinkIndexFails() throws Exception {
             assertTrue(routingTables.index("target").shard(0).shard(0).unassigned());
             assertEquals(
                 UnassignedInfo.Reason.ALLOCATION_FAILED,
-                routingTables.index("target").shard(0).shard(0).unassignedInfo().getReason()
+                routingTables.index("target").shard(0).shard(0).unassignedInfo().reason()
             );
-            assertEquals(1, routingTables.index("target").shard(0).shard(0).unassignedInfo().getNumFailedAllocations());
+            assertEquals(1, routingTables.index("target").shard(0).shard(0).unassignedInfo().failedAllocations());
         });
         // now relocate them all to the right node
         updateIndexSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode), "source");
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java
index 784a6e8f419c8..a25de555ce267 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/AllocationIdIT.java
@@ -113,7 +113,7 @@ public void testFailedRecoveryOnAllocateStalePrimaryRequiresAnotherAllocateStale
             final ClusterState state = clusterAdmin().prepareState().get().getState();
             final ShardRouting shardRouting = state.routingTable().index(indexName).shard(shardId.id()).primaryShard();
             assertThat(shardRouting.state(), equalTo(ShardRoutingState.UNASSIGNED));
-            assertThat(shardRouting.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
+            assertThat(shardRouting.unassignedInfo().reason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
         });
 
         internalCluster().stopNode(node1);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
index e7a7a6f2ba727..72594fef8c6ee 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
@@ -202,7 +202,7 @@ public void testFailedAllocationOfStalePrimaryToDataNodeWithNoData() throws Exce
                 .shard(0)
                 .primaryShard()
                 .unassignedInfo()
-                .getReason(),
+                .reason(),
             equalTo(UnassignedInfo.Reason.NODE_LEFT)
         );
 
@@ -227,7 +227,7 @@ public void testFailedAllocationOfStalePrimaryToDataNodeWithNoData() throws Exce
                 .shard(0)
                 .primaryShard()
                 .unassignedInfo()
-                .getReason(),
+                .reason(),
             equalTo(UnassignedInfo.Reason.NODE_LEFT)
         );
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java
index 006c9e2394f3c..76311387115d2 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java
@@ -114,7 +114,7 @@ public Collection<AllocationDecider> createAllocationDeciders(Settings settings,
                 @Override
                 public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
                     // once a primary is cancelled it _stays_ cancelled
-                    if (shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.REROUTE_CANCELLED) {
+                    if (shardRouting.unassignedInfo().reason() == UnassignedInfo.Reason.REROUTE_CANCELLED) {
                         return Decision.NO;
                     }
                     return super.canForceAllocatePrimary(shardRouting, node, allocation);
@@ -450,7 +450,7 @@ public AllocationCommand getCancelPrimaryCommand() {
                         shardRouting.role().isPromotableToPrimary()
                             ? UnassignedInfo.Reason.REROUTE_CANCELLED
                             : UnassignedInfo.Reason.UNPROMOTABLE_REPLICA,
-                        shardRouting.unassignedInfo().getReason()
+                        shardRouting.unassignedInfo().reason()
                     );
                 }
             }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index d1827bf49410f..e05bda69d2c9c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -403,9 +403,9 @@ public void testRecoverBrokenIndexMetadata() throws Exception {
                 assertTrue(shardRoutingTable.primaryShard().unassigned());
                 assertEquals(
                     UnassignedInfo.AllocationStatus.DECIDERS_NO,
-                    shardRoutingTable.primaryShard().unassignedInfo().getLastAllocationStatus()
+                    shardRoutingTable.primaryShard().unassignedInfo().lastAllocationStatus()
                 );
-                assertThat(shardRoutingTable.primaryShard().unassignedInfo().getNumFailedAllocations(), greaterThan(0));
+                assertThat(shardRoutingTable.primaryShard().unassignedInfo().failedAllocations(), greaterThan(0));
             }
         }, 60, TimeUnit.SECONDS);
         indicesAdmin().prepareClose("test").get();
@@ -472,9 +472,9 @@ public void testRecoverMissingAnalyzer() throws Exception {
                 assertTrue(shardRoutingTable.primaryShard().unassigned());
                 assertEquals(
                     UnassignedInfo.AllocationStatus.DECIDERS_NO,
-                    shardRoutingTable.primaryShard().unassignedInfo().getLastAllocationStatus()
+                    shardRoutingTable.primaryShard().unassignedInfo().lastAllocationStatus()
                 );
-                assertThat(shardRoutingTable.primaryShard().unassignedInfo().getNumFailedAllocations(), greaterThan(0));
+                assertThat(shardRoutingTable.primaryShard().unassignedInfo().failedAllocations(), greaterThan(0));
             }
         }, 60, TimeUnit.SECONDS);
         indicesAdmin().prepareClose("test").get();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java
index f43aaf0bacad4..ef4616fdd0b40 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java
@@ -311,8 +311,8 @@ public Settings onNodeStopped(String nodeName) throws Exception {
         // all shards should be failed due to a corrupted translog
         assertBusy(() -> {
             final UnassignedInfo unassignedInfo = getClusterAllocationExplanation(client(), indexName, 0, true).getUnassignedInfo();
-            assertThat(unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
-            assertThat(ExceptionsHelper.unwrap(unassignedInfo.getFailure(), TranslogCorruptedException.class), not(nullValue()));
+            assertThat(unassignedInfo.reason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
+            assertThat(ExceptionsHelper.unwrap(unassignedInfo.failure(), TranslogCorruptedException.class), not(nullValue()));
         });
 
         // have to shut down primary node - otherwise node lock is present
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java
index a9d19473164bf..7e3df8d8e1cbc 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -465,7 +465,7 @@ public void onTimeout(TimeValue timeout) {
                 final var replicaShards = indexRoutingTable.shard(shardId).replicaShards();
                 if (replicaShards.isEmpty()
                     || replicaShards.stream()
-                        .anyMatch(sr -> sr.unassigned() == false || sr.unassignedInfo().getNumFailedAllocations() < maxRetries)) {
+                        .anyMatch(sr -> sr.unassigned() == false || sr.unassignedInfo().failedAllocations() < maxRetries)) {
                     return false;
                 }
             }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
index ac5a10d246cfc..0c0ece4bf5227 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
@@ -83,8 +83,8 @@ public void onAllNodesStopped() throws Exception {
             final var description = Strings.toString(allocationExplainResponse);
             final var unassignedInfo = allocationExplainResponse.getUnassignedInfo();
             assertThat(description, unassignedInfo, not(nullValue()));
-            assertThat(description, unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
-            var failure = unassignedInfo.getFailure();
+            assertThat(description, unassignedInfo.reason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
+            var failure = unassignedInfo.failure();
             assertNotNull(failure);
             final Throwable cause = ExceptionsHelper.unwrap(failure, TranslogCorruptedException.class);
             if (cause != null) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java
index 59e7a67687921..874ba7b42690c 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java
@@ -61,7 +61,7 @@ public void testShardLockFailure() throws Exception {
                         .routingTable()
                         .shardRoutingTable(shardId)
                         .allShards()
-                        .noneMatch(sr -> sr.unassigned() && sr.unassignedInfo().getNumFailedAllocations() > 0)
+                        .noneMatch(sr -> sr.unassigned() && sr.unassignedInfo().failedAllocations() > 0)
                 );
             } catch (IndexNotFoundException e) {
                 // ok
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 005604b92a723..6e19cf60cf5b9 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -579,8 +579,8 @@ public void testUnrestorableFilesDuringRestore() throws Exception {
             .build();
 
         Consumer<UnassignedInfo> checkUnassignedInfo = unassignedInfo -> {
-            assertThat(unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
-            assertThat(unassignedInfo.getNumFailedAllocations(), anyOf(equalTo(maxRetries), equalTo(1)));
+            assertThat(unassignedInfo.reason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
+            assertThat(unassignedInfo.failedAllocations(), anyOf(equalTo(maxRetries), equalTo(1)));
         };
 
         unrestorableUseCase(indexName, createIndexSettings, repositorySettings, Settings.EMPTY, checkUnassignedInfo, () -> {});
@@ -605,7 +605,7 @@ public void testUnrestorableIndexDuringRestore() throws Exception {
             Settings.EMPTY,
             Settings.EMPTY,
             restoreIndexSettings,
-            unassignedInfo -> assertThat(unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.NEW_INDEX_RESTORED)),
+            unassignedInfo -> assertThat(unassignedInfo.reason(), equalTo(UnassignedInfo.Reason.NEW_INDEX_RESTORED)),
             fixupAction
         );
     }
@@ -670,7 +670,7 @@ private void unrestorableUseCase(
             if (shard.primary()) {
                 assertThat(shard.state(), equalTo(ShardRoutingState.UNASSIGNED));
                 assertThat(shard.recoverySource().getType(), equalTo(RecoverySource.Type.SNAPSHOT));
-                assertThat(shard.unassignedInfo().getLastAllocationStatus(), equalTo(UnassignedInfo.AllocationStatus.DECIDERS_NO));
+                assertThat(shard.unassignedInfo().lastAllocationStatus(), equalTo(UnassignedInfo.AllocationStatus.DECIDERS_NO));
                 checkUnassignedInfo.accept(shard.unassignedInfo());
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
index d22bae9c5a4b1..1e5f9d5d613d2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
@@ -226,16 +226,16 @@ private Iterator<? extends ToXContent> getShardAllocationDecisionChunked(ToXCont
     private static XContentBuilder unassignedInfoToXContent(UnassignedInfo unassignedInfo, XContentBuilder builder) throws IOException {
 
         builder.startObject("unassigned_info");
-        builder.field("reason", unassignedInfo.getReason());
-        builder.field("at", UnassignedInfo.DATE_TIME_FORMATTER.format(Instant.ofEpochMilli(unassignedInfo.getUnassignedTimeInMillis())));
-        if (unassignedInfo.getNumFailedAllocations() > 0) {
-            builder.field("failed_allocation_attempts", unassignedInfo.getNumFailedAllocations());
+        builder.field("reason", unassignedInfo.reason());
+        builder.field("at", UnassignedInfo.DATE_TIME_FORMATTER.format(Instant.ofEpochMilli(unassignedInfo.unassignedTimeMillis())));
+        if (unassignedInfo.failedAllocations() > 0) {
+            builder.field("failed_allocation_attempts", unassignedInfo.failedAllocations());
         }
-        String details = unassignedInfo.getDetails();
+        String details = unassignedInfo.details();
         if (details != null) {
             builder.field("details", details);
         }
-        builder.field("last_allocation_status", AllocationDecision.fromAllocationStatus(unassignedInfo.getLastAllocationStatus()));
+        builder.field("last_allocation_status", AllocationDecision.fromAllocationStatus(unassignedInfo.lastAllocationStatus()));
         builder.endObject();
         return builder;
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java
index 785b0db5cc807..adb5a7caf2f45 100644
--- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java
+++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java
@@ -167,8 +167,8 @@ public static ClusterHealthStatus getInactivePrimaryHealth(final ShardRouting sh
         assert shardRouting.recoverySource() != null : "cannot invoke on a shard that has no recovery source" + shardRouting;
         final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
         RecoverySource.Type recoveryType = shardRouting.recoverySource().getType();
-        if (unassignedInfo.getLastAllocationStatus() != AllocationStatus.DECIDERS_NO
-            && unassignedInfo.getNumFailedAllocations() == 0
+        if (unassignedInfo.lastAllocationStatus() != AllocationStatus.DECIDERS_NO
+            && unassignedInfo.failedAllocations() == 0
             && (recoveryType == RecoverySource.Type.EMPTY_STORE
                 || recoveryType == RecoverySource.Type.LOCAL_SHARDS
                 || recoveryType == RecoverySource.Type.SNAPSHOT)) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
index 6679f17a0427b..d62dd91d7e87b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
@@ -574,15 +574,15 @@ private static UnassignedInfo withLastAllocatedNodeId(UnassignedInfo unassignedI
             return previousNodes == null || previousNodes.size() <= shardCopy
                 ? unassignedInfo
                 : new UnassignedInfo(
-                    unassignedInfo.getReason(),
-                    unassignedInfo.getMessage(),
-                    unassignedInfo.getFailure(),
-                    unassignedInfo.getNumFailedAllocations(),
-                    unassignedInfo.getUnassignedTimeInNanos(),
-                    unassignedInfo.getUnassignedTimeInMillis(),
-                    unassignedInfo.isDelayed(),
-                    unassignedInfo.getLastAllocationStatus(),
-                    unassignedInfo.getFailedNodeIds(),
+                    unassignedInfo.reason(),
+                    unassignedInfo.message(),
+                    unassignedInfo.failure(),
+                    unassignedInfo.failedAllocations(),
+                    unassignedInfo.unassignedTimeNanos(),
+                    unassignedInfo.unassignedTimeMillis(),
+                    unassignedInfo.delayed(),
+                    unassignedInfo.lastAllocationStatus(),
+                    unassignedInfo.failedNodeIds(),
                     previousNodes.get(shardCopy)
                 );
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index cf8e0608ecbd4..0b3cadb6e187c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -563,8 +563,8 @@ assert getByAllocationId(failedShard.shardId(), failedShard.allocationId().getId
                             "primary failed while replica initializing",
                             null,
                             0,
-                            unassignedInfo.getUnassignedTimeInNanos(),
-                            unassignedInfo.getUnassignedTimeInMillis(),
+                            unassignedInfo.unassignedTimeNanos(),
+                            unassignedInfo.unassignedTimeMillis(),
                             false,
                             AllocationStatus.NO_ATTEMPT,
                             Collections.emptySet(),
@@ -644,11 +644,11 @@ private void unassignPrimaryAndPromoteActiveReplicaIfExists(
                     unpromotableReplica,
                     new UnassignedInfo(
                         UnassignedInfo.Reason.UNPROMOTABLE_REPLICA,
-                        unassignedInfo.getMessage(),
-                        unassignedInfo.getFailure(),
+                        unassignedInfo.message(),
+                        unassignedInfo.failure(),
                         0,
-                        unassignedInfo.getUnassignedTimeInNanos(),
-                        unassignedInfo.getUnassignedTimeInMillis(),
+                        unassignedInfo.unassignedTimeNanos(),
+                        unassignedInfo.unassignedTimeMillis(),
                         false, // TODO debatable, but do we want to delay reassignment of unpromotable replicas tho?
                         AllocationStatus.NO_ATTEMPT,
                         Set.of(),
@@ -970,18 +970,18 @@ public void ignoreShard(ShardRouting shard, AllocationStatus allocationStatus, R
                 ignoredPrimaries++;
                 UnassignedInfo currInfo = shard.unassignedInfo();
                 assert currInfo != null;
-                if (allocationStatus.equals(currInfo.getLastAllocationStatus()) == false) {
+                if (allocationStatus.equals(currInfo.lastAllocationStatus()) == false) {
                     UnassignedInfo newInfo = new UnassignedInfo(
-                        currInfo.getReason(),
-                        currInfo.getMessage(),
-                        currInfo.getFailure(),
-                        currInfo.getNumFailedAllocations(),
-                        currInfo.getUnassignedTimeInNanos(),
-                        currInfo.getUnassignedTimeInMillis(),
-                        currInfo.isDelayed(),
+                        currInfo.reason(),
+                        currInfo.message(),
+                        currInfo.failure(),
+                        currInfo.failedAllocations(),
+                        currInfo.unassignedTimeNanos(),
+                        currInfo.unassignedTimeMillis(),
+                        currInfo.delayed(),
                         allocationStatus,
-                        currInfo.getFailedNodeIds(),
-                        currInfo.getLastAllocatedNodeId()
+                        currInfo.failedNodeIds(),
+                        currInfo.lastAllocatedNodeId()
                     );
                     ShardRouting updatedShard = shard.updateUnassigned(newInfo, shard.recoverySource());
                     changes.unassignedInfoUpdated(shard, newInfo);
@@ -1283,16 +1283,16 @@ public void resetFailedCounter(RoutingChangesObserver routingChangesObserver) {
             UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
             unassignedIterator.updateUnassigned(
                 new UnassignedInfo(
-                    unassignedInfo.getNumFailedAllocations() > 0 ? UnassignedInfo.Reason.MANUAL_ALLOCATION : unassignedInfo.getReason(),
-                    unassignedInfo.getMessage(),
-                    unassignedInfo.getFailure(),
+                    unassignedInfo.failedAllocations() > 0 ? UnassignedInfo.Reason.MANUAL_ALLOCATION : unassignedInfo.reason(),
+                    unassignedInfo.message(),
+                    unassignedInfo.failure(),
                     0,
-                    unassignedInfo.getUnassignedTimeInNanos(),
-                    unassignedInfo.getUnassignedTimeInMillis(),
-                    unassignedInfo.isDelayed(),
-                    unassignedInfo.getLastAllocationStatus(),
+                    unassignedInfo.unassignedTimeNanos(),
+                    unassignedInfo.unassignedTimeMillis(),
+                    unassignedInfo.delayed(),
+                    unassignedInfo.lastAllocationStatus(),
                     Collections.emptySet(),
-                    unassignedInfo.getLastAllocatedNodeId()
+                    unassignedInfo.lastAllocatedNodeId()
                 ),
                 shardRouting.recoverySource(),
                 routingChangesObserver
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
index 95882e26773e5..523dc0efd450b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
@@ -342,7 +342,7 @@ public ShardRouting(ShardId shardId, StreamInput in) throws IOException {
         } else {
             recoverySource = null;
         }
-        unassignedInfo = in.readOptionalWriteable(UnassignedInfo::new);
+        unassignedInfo = in.readOptionalWriteable(UnassignedInfo::fromStreamInput);
         if (in.getTransportVersion().onOrAfter(RELOCATION_FAILURE_INFO_VERSION)) {
             relocationFailureInfo = RelocationFailureInfo.readFrom(in);
         } else {
@@ -410,7 +410,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     public ShardRouting updateUnassigned(UnassignedInfo unassignedInfo, RecoverySource recoverySource) {
         assert this.unassignedInfo != null : "can only update unassigned info if it is already set";
-        assert this.unassignedInfo.isDelayed() || (unassignedInfo.isDelayed() == false) : "cannot transition from non-delayed to delayed";
+        assert this.unassignedInfo.delayed() || (unassignedInfo.delayed() == false) : "cannot transition from non-delayed to delayed";
         return new ShardRouting(
             shardId,
             currentNodeId,
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
index bde667df3821a..9423e32be6846 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
@@ -41,9 +41,40 @@
 import static org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING;
 
 /**
- * Holds additional information as to why the shard is in unassigned state.
+ * Holds additional information as to why the shard is in an unassigned state.
+ *
+ * @param reason why the shard is unassigned.
+ * @param message optional details explaining the reasons.
+ * @param failure additional failure exception details if exists.
+ * @param failedAllocations number of previously failed allocations of this shard.
+ * @param delayed true if allocation of this shard is delayed due to {@link #INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING}.
+ * @param unassignedTimeMillis The timestamp in milliseconds when the shard became unassigned, based on System.currentTimeMillis().
+ *                             Note, we use timestamp here since we want to make sure its preserved across node serializations.
+ * @param unassignedTimeNanos The timestamp in nanoseconds when the shard became unassigned, based on System.nanoTime().
+ *                            Used to calculate the delay for delayed shard allocation.
+ *                            ONLY EXPOSED FOR TESTS!
+ * @param lastAllocationStatus status for the last allocation attempt for this shard.
+ * @param failedNodeIds A set of nodeIds that failed to complete allocations for this shard.
+ *                      {@link org.elasticsearch.gateway.ReplicaShardAllocator} uses this bset to avoid repeatedly canceling ongoing
+ *                      recoveries for copies on those nodes, although they can perform noop recoveries. This set will be discarded when a
+ *                      shard moves to started. And if a shard is failed while started (i.e., from started to unassigned), the currently
+ *                      assigned node won't be added to this set.
+ *                      @see org.elasticsearch.gateway.ReplicaShardAllocator#processExistingRecoveries
+ *                      @see org.elasticsearch.cluster.routing.allocation.AllocationService#applyFailedShards(ClusterState, List, List)
+ * @param lastAllocatedNodeId ID of the node this shard was last allocated to, or null if unavailable.
  */
-public final class UnassignedInfo implements ToXContentFragment, Writeable {
+public record UnassignedInfo(
+    Reason reason,
+    @Nullable String message,
+    @Nullable Exception failure,
+    int failedAllocations,
+    long unassignedTimeNanos,
+    long unassignedTimeMillis,
+    boolean delayed,
+    AllocationStatus lastAllocationStatus,
+    Set<String> failedNodeIds,
+    @Nullable String lastAllocatedNodeId
+) implements ToXContentFragment, Writeable {
 
     /**
      * The version that the {@code lastAllocatedNode} field was added in. Used to adapt streaming of this class as appropriate for the
@@ -218,17 +249,6 @@ public String value() {
         }
     }
 
-    private final Reason reason;
-    private final long unassignedTimeMillis; // used for display and log messages, in milliseconds
-    private final long unassignedTimeNanos; // in nanoseconds, used to calculate delay for delayed shard allocation
-    private final boolean delayed; // if allocation of this shard is delayed due to INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING
-    private final String message;
-    private final Exception failure;
-    private final int failedAllocations;
-    private final Set<String> failedNodeIds;
-    private final AllocationStatus lastAllocationStatus; // result of the last allocation attempt for this shard
-    private final String lastAllocatedNodeId;
-
     /**
      * creates an UnassignedInfo object based on **current** time
      *
@@ -261,28 +281,10 @@ public UnassignedInfo(Reason reason, String message) {
      * @param failedNodeIds                   a set of nodeIds that failed to complete allocations for this shard
      * @param lastAllocatedNodeId             the ID of the node this shard was last allocated to
      */
-    public UnassignedInfo(
-        Reason reason,
-        @Nullable String message,
-        @Nullable Exception failure,
-        int failedAllocations,
-        long unassignedTimeNanos,
-        long unassignedTimeMillis,
-        boolean delayed,
-        AllocationStatus lastAllocationStatus,
-        Set<String> failedNodeIds,
-        @Nullable String lastAllocatedNodeId
-    ) {
-        this.reason = Objects.requireNonNull(reason);
-        this.unassignedTimeMillis = unassignedTimeMillis;
-        this.unassignedTimeNanos = unassignedTimeNanos;
-        this.delayed = delayed;
-        this.message = message;
-        this.failure = failure;
-        this.failedAllocations = failedAllocations;
-        this.lastAllocationStatus = Objects.requireNonNull(lastAllocationStatus);
-        this.failedNodeIds = Set.copyOf(failedNodeIds);
-        this.lastAllocatedNodeId = lastAllocatedNodeId;
+    public UnassignedInfo {
+        Objects.requireNonNull(reason);
+        Objects.requireNonNull(lastAllocationStatus);
+        failedNodeIds = Set.copyOf(failedNodeIds);
         assert (failedAllocations > 0) == (reason == Reason.ALLOCATION_FAILED)
             : "failedAllocations: " + failedAllocations + " for reason " + reason;
         assert (message == null && failure != null) == false : "provide a message if a failure exception is provided";
@@ -294,24 +296,37 @@ public UnassignedInfo(
             : "last allocated node ID must be set if the shard is unassigned due to a node restarting";
     }
 
-    public UnassignedInfo(StreamInput in) throws IOException {
+    public static UnassignedInfo fromStreamInput(StreamInput in) throws IOException {
         // Because Reason.NODE_RESTARTING is new and can't be sent by older versions, there's no need to vary the deserialization behavior
-        this.reason = Reason.values()[(int) in.readByte()];
-        this.unassignedTimeMillis = in.readLong();
+        var reason = Reason.values()[(int) in.readByte()];
+        var unassignedTimeMillis = in.readLong();
         // As System.nanoTime() cannot be compared across different JVMs, reset it to now.
         // This means that in master fail-over situations, elapsed delay time is forgotten.
-        this.unassignedTimeNanos = System.nanoTime();
-        this.delayed = in.readBoolean();
-        this.message = in.readOptionalString();
-        this.failure = in.readException();
-        this.failedAllocations = in.readVInt();
-        this.lastAllocationStatus = AllocationStatus.readFrom(in);
-        this.failedNodeIds = in.readCollectionAsImmutableSet(StreamInput::readString);
+        var unassignedTimeNanos = System.nanoTime();
+        var delayed = in.readBoolean();
+        var message = in.readOptionalString();
+        var failure = in.readException();
+        var failedAllocations = in.readVInt();
+        var lastAllocationStatus = AllocationStatus.readFrom(in);
+        var failedNodeIds = in.readCollectionAsImmutableSet(StreamInput::readString);
+        String lastAllocatedNodeId;
         if (in.getTransportVersion().onOrAfter(VERSION_LAST_ALLOCATED_NODE_ADDED)) {
-            this.lastAllocatedNodeId = in.readOptionalString();
+            lastAllocatedNodeId = in.readOptionalString();
         } else {
-            this.lastAllocatedNodeId = null;
+            lastAllocatedNodeId = null;
         }
+        return new UnassignedInfo(
+            reason,
+            message,
+            failure,
+            failedAllocations,
+            unassignedTimeNanos,
+            unassignedTimeMillis,
+            delayed,
+            lastAllocationStatus,
+            failedNodeIds,
+            lastAllocatedNodeId
+        );
     }
 
     public void writeTo(StreamOutput out) throws IOException {
@@ -335,107 +350,25 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    /**
-     * Returns the number of previously failed allocations of this shard.
-     */
-    public int getNumFailedAllocations() {
-        return failedAllocations;
-    }
-
-    /**
-     * Returns true if allocation of this shard is delayed due to {@link #INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING}
-     */
-    public boolean isDelayed() {
-        return delayed;
-    }
-
-    /**
-     * The reason why the shard is unassigned.
-     */
-    public Reason getReason() {
-        return this.reason;
-    }
-
-    /**
-     * The timestamp in milliseconds when the shard became unassigned, based on System.currentTimeMillis().
-     * Note, we use timestamp here since we want to make sure its preserved across node serializations.
-     */
-    public long getUnassignedTimeInMillis() {
-        return this.unassignedTimeMillis;
-    }
-
-    /**
-     * The timestamp in nanoseconds when the shard became unassigned, based on System.nanoTime().
-     * Used to calculate the delay for delayed shard allocation.
-     * ONLY EXPOSED FOR TESTS!
-     */
-    public long getUnassignedTimeInNanos() {
-        return this.unassignedTimeNanos;
-    }
-
-    /**
-     * Returns optional details explaining the reasons.
-     */
-    @Nullable
-    public String getMessage() {
-        return this.message;
-    }
-
-    /**
-     * Returns additional failure exception details if exists.
-     */
-    @Nullable
-    public Exception getFailure() {
-        return failure;
-    }
-
     /**
      * Builds a string representation of the message and the failure if exists.
      */
     @Nullable
-    public String getDetails() {
+    public String details() {
         if (message == null) {
             return null;
         }
         return message + (failure == null ? "" : ", failure " + ExceptionsHelper.stackTrace(failure));
     }
 
-    /**
-     * Gets the ID of the node this shard was last allocated to, or null if unavailable.
-     */
-    @Nullable
-    public String getLastAllocatedNodeId() {
-        return lastAllocatedNodeId;
-    }
-
-    /**
-     * Get the status for the last allocation attempt for this shard.
-     */
-    public AllocationStatus getLastAllocationStatus() {
-        return lastAllocationStatus;
-    }
-
-    /**
-     * A set of nodeIds that failed to complete allocations for this shard. {@link org.elasticsearch.gateway.ReplicaShardAllocator}
-     * uses this set to avoid repeatedly canceling ongoing recoveries for copies on those nodes although they can perform noop recoveries.
-     * This set will be discarded when a shard moves to started. And if a shard is failed while started (i.e., from started to unassigned),
-     * the currently assigned node won't be added to this set.
-     *
-     * @see org.elasticsearch.gateway.ReplicaShardAllocator#processExistingRecoveries
-     * @see org.elasticsearch.cluster.routing.allocation.AllocationService#applyFailedShards(ClusterState, List, List)
-     */
-    public Set<String> getFailedNodeIds() {
-        return failedNodeIds;
-    }
-
     /**
      * Calculates the delay left based on current time (in nanoseconds) and the delay defined by the index settings.
-     * Only relevant if shard is effectively delayed (see {@link #isDelayed()})
+     * Only relevant if shard is effectively delayed (see {@link #delayed()})
      * Returns 0 if delay is negative
      *
      * @return calculated delay in nanoseconds
      */
-    public long getRemainingDelay(final long nanoTimeNow, final Settings indexSettings, final NodesShutdownMetadata nodesShutdownMetadata) {
+    public long remainingDelay(final long nanoTimeNow, final Settings indexSettings, final NodesShutdownMetadata nodesShutdownMetadata) {
         final long indexLevelDelay = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexSettings).nanos();
         long delayTimeoutNanos = Optional.ofNullable(lastAllocatedNodeId)
             // If the node wasn't restarting when this became unassigned, use default delay
@@ -455,7 +388,7 @@ public long getRemainingDelay(final long nanoTimeNow, final Settings indexSettin
     public static int getNumberOfDelayedUnassigned(ClusterState state) {
         int count = 0;
         for (ShardRouting shard : state.getRoutingNodes().unassigned()) {
-            if (shard.unassignedInfo().isDelayed()) {
+            if (shard.unassignedInfo().delayed()) {
                 count++;
             }
         }
@@ -472,10 +405,10 @@ public static long findNextDelayedAllocation(long currentNanoTime, ClusterState
         long nextDelayNanos = Long.MAX_VALUE;
         for (ShardRouting shard : state.getRoutingNodes().unassigned()) {
             UnassignedInfo unassignedInfo = shard.unassignedInfo();
-            if (unassignedInfo.isDelayed()) {
+            if (unassignedInfo.delayed()) {
                 Settings indexSettings = metadata.index(shard.index()).getSettings();
                 // calculate next time to schedule
-                final long newComputedLeftDelayNanos = unassignedInfo.getRemainingDelay(
+                final long newComputedLeftDelayNanos = unassignedInfo.remainingDelay(
                     currentNanoTime,
                     indexSettings,
                     metadata.nodeShutdowns()
@@ -502,7 +435,7 @@ public String shortSummary() {
         if (lastAllocatedNodeId != null) {
             sb.append(", last_node[").append(lastAllocatedNodeId).append("]");
         }
-        String details = getDetails();
+        String details = details();
         if (details != null) {
             sb.append(", details[").append(details).append("]");
         }
@@ -530,7 +463,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (lastAllocatedNodeId != null) {
             builder.field("last_node", lastAllocatedNodeId);
         }
-        String details = getDetails();
+        String details = details();
         if (details != null) {
             builder.field("details", details);
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index e19e266cc2672..436399a02005f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -215,11 +215,11 @@ public ClusterState applyFailedShards(
                         failedShard
                     );
                 }
-                int failedAllocations = failedShard.unassignedInfo() != null ? failedShard.unassignedInfo().getNumFailedAllocations() : 0;
+                int failedAllocations = failedShard.unassignedInfo() != null ? failedShard.unassignedInfo().failedAllocations() : 0;
                 final Set<String> failedNodeIds;
                 if (failedShard.unassignedInfo() != null) {
-                    failedNodeIds = Sets.newHashSetWithExpectedSize(failedShard.unassignedInfo().getFailedNodeIds().size() + 1);
-                    failedNodeIds.addAll(failedShard.unassignedInfo().getFailedNodeIds());
+                    failedNodeIds = Sets.newHashSetWithExpectedSize(failedShard.unassignedInfo().failedNodeIds().size() + 1);
+                    failedNodeIds.addAll(failedShard.unassignedInfo().failedNodeIds());
                     failedNodeIds.add(failedShard.currentNodeId());
                 } else {
                     failedNodeIds = Collections.emptySet();
@@ -425,8 +425,8 @@ default void removeDelayMarkers(RoutingAllocation allocation) {
             while (unassignedIterator.hasNext()) {
                 ShardRouting shardRouting = unassignedIterator.next();
                 UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
-                if (unassignedInfo.isDelayed()) {
-                    final long newComputedLeftDelayNanos = unassignedInfo.getRemainingDelay(
+                if (unassignedInfo.delayed()) {
+                    final long newComputedLeftDelayNanos = unassignedInfo.remainingDelay(
                         allocation.getCurrentNanoTime(),
                         metadata.getIndexSafe(shardRouting.index()).getSettings(),
                         metadata.nodeShutdowns()
@@ -434,16 +434,16 @@ default void removeDelayMarkers(RoutingAllocation allocation) {
                     if (newComputedLeftDelayNanos == 0) {
                         unassignedIterator.updateUnassigned(
                             new UnassignedInfo(
-                                unassignedInfo.getReason(),
-                                unassignedInfo.getMessage(),
-                                unassignedInfo.getFailure(),
-                                unassignedInfo.getNumFailedAllocations(),
-                                unassignedInfo.getUnassignedTimeInNanos(),
-                                unassignedInfo.getUnassignedTimeInMillis(),
+                                unassignedInfo.reason(),
+                                unassignedInfo.message(),
+                                unassignedInfo.failure(),
+                                unassignedInfo.failedAllocations(),
+                                unassignedInfo.unassignedTimeNanos(),
+                                unassignedInfo.unassignedTimeMillis(),
                                 false,
-                                unassignedInfo.getLastAllocationStatus(),
-                                unassignedInfo.getFailedNodeIds(),
-                                unassignedInfo.getLastAllocatedNodeId()
+                                unassignedInfo.lastAllocationStatus(),
+                                unassignedInfo.failedNodeIds(),
+                                unassignedInfo.lastAllocatedNodeId()
                             ),
                             shardRouting.recoverySource(),
                             allocation.changes()
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserver.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserver.java
index 1b5d1875bc1d3..f265ab7f62db2 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardChangesObserver.java
@@ -36,7 +36,7 @@ public void relocationStarted(ShardRouting startedShard, ShardRouting targetRelo
 
     @Override
     public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) {
-        logger.debug("{} has failed on [{}]: {}", shardIdentifier(failedShard), failedShard.currentNodeId(), unassignedInfo.getReason());
+        logger.debug("{} has failed on [{}]: {}", shardIdentifier(failedShard), failedShard.currentNodeId(), unassignedInfo.reason());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 57f28e4ea021c..2fca8895b011c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -220,19 +220,19 @@ private void failAllocationOfNewPrimaries(RoutingAllocation allocation) {
         while (unassignedIterator.hasNext()) {
             final ShardRouting shardRouting = unassignedIterator.next();
             final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
-            if (shardRouting.primary() && unassignedInfo.getLastAllocationStatus() == AllocationStatus.NO_ATTEMPT) {
+            if (shardRouting.primary() && unassignedInfo.lastAllocationStatus() == AllocationStatus.NO_ATTEMPT) {
                 unassignedIterator.updateUnassigned(
                     new UnassignedInfo(
-                        unassignedInfo.getReason(),
-                        unassignedInfo.getMessage(),
-                        unassignedInfo.getFailure(),
-                        unassignedInfo.getNumFailedAllocations(),
-                        unassignedInfo.getUnassignedTimeInNanos(),
-                        unassignedInfo.getUnassignedTimeInMillis(),
-                        unassignedInfo.isDelayed(),
+                        unassignedInfo.reason(),
+                        unassignedInfo.message(),
+                        unassignedInfo.failure(),
+                        unassignedInfo.failedAllocations(),
+                        unassignedInfo.unassignedTimeNanos(),
+                        unassignedInfo.unassignedTimeMillis(),
+                        unassignedInfo.delayed(),
                         AllocationStatus.DECIDERS_NO,
-                        unassignedInfo.getFailedNodeIds(),
-                        unassignedInfo.getLastAllocatedNodeId()
+                        unassignedInfo.failedNodeIds(),
+                        unassignedInfo.lastAllocatedNodeId()
                     ),
                     shardRouting.recoverySource(),
                     allocation.changes()
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java
index f0fd108dd31fd..7c04d518eb2f6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java
@@ -120,7 +120,7 @@ public DesiredBalance compute(
             for (final var iterator = unassigned.iterator(); iterator.hasNext();) {
                 final var shardRouting = iterator.next();
                 if (shardRouting.primary() == primary) {
-                    var lastAllocatedNodeId = shardRouting.unassignedInfo().getLastAllocatedNodeId();
+                    var lastAllocatedNodeId = shardRouting.unassignedInfo().lastAllocatedNodeId();
                     if (knownNodeIds.contains(lastAllocatedNodeId)
                         || ignoredShards.contains(discardAllocationStatus(shardRouting)) == false) {
                         shardRoutings.computeIfAbsent(shardRouting.shardId(), ShardRoutings::new).unassigned().add(shardRouting);
@@ -154,7 +154,7 @@ public DesiredBalance compute(
 
             // preserving last known shard location as a starting point to avoid unnecessary relocations
             for (ShardRouting shardRouting : routings.unassigned()) {
-                var lastAllocatedNodeId = shardRouting.unassignedInfo().getLastAllocatedNodeId();
+                var lastAllocatedNodeId = shardRouting.unassignedInfo().lastAllocatedNodeId();
                 if (knownNodeIds.contains(lastAllocatedNodeId)) {
                     targetNodes.add(lastAllocatedNodeId);
                 }
@@ -346,19 +346,18 @@ public DesiredBalance compute(
         for (var shard : routingNodes.unassigned().ignored()) {
             var info = shard.unassignedInfo();
             assert info != null
-                && (info.getLastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO
-                    || info.getLastAllocationStatus() == UnassignedInfo.AllocationStatus.NO_ATTEMPT
-                    || info.getLastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_THROTTLED)
-                : "Unexpected stats in: " + info;
+                && (info.lastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO
+                    || info.lastAllocationStatus() == UnassignedInfo.AllocationStatus.NO_ATTEMPT
+                    || info.lastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_THROTTLED) : "Unexpected stats in: " + info;
 
-            if (hasChanges == false && info.getLastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_THROTTLED) {
+            if (hasChanges == false && info.lastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_THROTTLED) {
                 // Simulation could not progress due to missing information in any of the deciders.
                 // Currently, this could happen if `HasFrozenCacheAllocationDecider` is still fetching the data.
                 // Progress would be made after the followup reroute call.
                 hasChanges = true;
             }
 
-            var ignored = shard.unassignedInfo().getLastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO ? 0 : 1;
+            var ignored = shard.unassignedInfo().lastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO ? 0 : 1;
             assignments.compute(
                 shard.shardId(),
                 (key, oldValue) -> oldValue == null
@@ -400,20 +399,20 @@ private static ShardRouting discardAllocationStatus(ShardRouting shardRouting) {
     }
 
     private static UnassignedInfo discardAllocationStatus(UnassignedInfo info) {
-        if (info.getLastAllocationStatus() == UnassignedInfo.AllocationStatus.NO_ATTEMPT) {
+        if (info.lastAllocationStatus() == UnassignedInfo.AllocationStatus.NO_ATTEMPT) {
             return info;
         }
         return new UnassignedInfo(
-            info.getReason(),
-            info.getMessage(),
-            info.getFailure(),
-            info.getNumFailedAllocations(),
-            info.getUnassignedTimeInNanos(),
-            info.getUnassignedTimeInMillis(),
-            info.isDelayed(),
+            info.reason(),
+            info.message(),
+            info.failure(),
+            info.failedAllocations(),
+            info.unassignedTimeNanos(),
+            info.unassignedTimeMillis(),
+            info.delayed(),
             UnassignedInfo.AllocationStatus.NO_ATTEMPT,
-            info.getFailedNodeIds(),
-            info.getLastAllocatedNodeId()
+            info.failedNodeIds(),
+            info.lastAllocatedNodeId()
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
index f70d0b8929252..24e7abca45d2d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
@@ -226,19 +226,19 @@ private void failAllocationOfNewPrimaries(RoutingAllocation allocation) {
             while (unassignedIterator.hasNext()) {
                 final ShardRouting shardRouting = unassignedIterator.next();
                 final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
-                if (shardRouting.primary() && unassignedInfo.getLastAllocationStatus() == AllocationStatus.NO_ATTEMPT) {
+                if (shardRouting.primary() && unassignedInfo.lastAllocationStatus() == AllocationStatus.NO_ATTEMPT) {
                     unassignedIterator.updateUnassigned(
                         new UnassignedInfo(
-                            unassignedInfo.getReason(),
-                            unassignedInfo.getMessage(),
-                            unassignedInfo.getFailure(),
-                            unassignedInfo.getNumFailedAllocations(),
-                            unassignedInfo.getUnassignedTimeInNanos(),
-                            unassignedInfo.getUnassignedTimeInMillis(),
-                            unassignedInfo.isDelayed(),
+                            unassignedInfo.reason(),
+                            unassignedInfo.message(),
+                            unassignedInfo.failure(),
+                            unassignedInfo.failedAllocations(),
+                            unassignedInfo.unassignedTimeNanos(),
+                            unassignedInfo.unassignedTimeMillis(),
+                            unassignedInfo.delayed(),
                             AllocationStatus.DECIDERS_NO,
-                            unassignedInfo.getFailedNodeIds(),
-                            unassignedInfo.getLastAllocatedNodeId()
+                            unassignedInfo.failedNodeIds(),
+                            unassignedInfo.lastAllocatedNodeId()
                         ),
                         shardRouting.recoverySource(),
                         allocation.changes()
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
index e2fdec43d8e12..2b006988a2ae4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
@@ -126,20 +126,20 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain)
         }
 
         UnassignedInfo unassignedInfoToUpdate = null;
-        if (shardRouting.unassignedInfo().getReason() != UnassignedInfo.Reason.FORCED_EMPTY_PRIMARY) {
+        if (shardRouting.unassignedInfo().reason() != UnassignedInfo.Reason.FORCED_EMPTY_PRIMARY) {
             String unassignedInfoMessage = "force empty allocation from previous reason "
-                + shardRouting.unassignedInfo().getReason()
+                + shardRouting.unassignedInfo().reason()
                 + ", "
-                + shardRouting.unassignedInfo().getMessage();
+                + shardRouting.unassignedInfo().message();
             unassignedInfoToUpdate = new UnassignedInfo(
                 UnassignedInfo.Reason.FORCED_EMPTY_PRIMARY,
                 unassignedInfoMessage,
-                shardRouting.unassignedInfo().getFailure(),
+                shardRouting.unassignedInfo().failure(),
                 0,
                 System.nanoTime(),
                 System.currentTimeMillis(),
                 false,
-                shardRouting.unassignedInfo().getLastAllocationStatus(),
+                shardRouting.unassignedInfo().lastAllocationStatus(),
                 Collections.emptySet(),
                 null
             );
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
index f37039608d7bd..1f7d1fe0143c3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
@@ -50,7 +50,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
 
         final int maxRetries = SETTING_ALLOCATION_MAX_RETRY.get(allocation.metadata().getIndexSafe(shardRouting.index()).getSettings());
         final var unassignedInfo = shardRouting.unassignedInfo();
-        final int numFailedAllocations = unassignedInfo == null ? 0 : unassignedInfo.getNumFailedAllocations();
+        final int numFailedAllocations = unassignedInfo == null ? 0 : unassignedInfo.failedAllocations();
         if (numFailedAllocations > 0) {
             final var decision = numFailedAllocations >= maxRetries ? Decision.NO : Decision.YES;
             return allocation.debugDecision() ? debugDecision(decision, unassignedInfo, numFailedAllocations, maxRetries) : decision;
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java
index 7b08a4d94512e..7adfc2c17d4aa 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java
@@ -55,7 +55,7 @@ public Decision canAllocate(final ShardRouting shardRouting, final RoutingAlloca
                 + "to restore the snapshot again or use the reroute API to force the allocation of an empty primary shard. Details: [%s]",
             source.snapshot(),
             shardRouting.getIndexName(),
-            shardRouting.unassignedInfo().getDetails()
+            shardRouting.unassignedInfo().details()
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java
index 309848635a440..8fb91d89417e0 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorService.java
@@ -536,15 +536,15 @@ static boolean isNewlyCreatedAndInitializingReplica(ShardRouting routing, Cluste
 
     private static boolean isUnassignedDueToTimelyRestart(ShardRouting routing, NodesShutdownMetadata shutdowns) {
         var info = routing.unassignedInfo();
-        if (info == null || info.getReason() != UnassignedInfo.Reason.NODE_RESTARTING) {
+        if (info == null || info.reason() != UnassignedInfo.Reason.NODE_RESTARTING) {
             return false;
         }
-        var shutdown = shutdowns.get(info.getLastAllocatedNodeId(), SingleNodeShutdownMetadata.Type.RESTART);
+        var shutdown = shutdowns.get(info.lastAllocatedNodeId(), SingleNodeShutdownMetadata.Type.RESTART);
         if (shutdown == null) {
             return false;
         }
         var now = System.nanoTime();
-        var restartingAllocationDelayExpiration = info.getUnassignedTimeInNanos() + shutdown.getAllocationDelay().nanos();
+        var restartingAllocationDelayExpiration = info.unassignedTimeNanos() + shutdown.getAllocationDelay().nanos();
         return now - restartingAllocationDelayExpiration <= 0;
     }
 
@@ -567,10 +567,10 @@ private static boolean isUnassignedDueToNewInitialization(ShardRouting routing,
     List<Diagnosis.Definition> diagnoseUnassignedShardRouting(ShardRouting shardRouting, ClusterState state) {
         List<Diagnosis.Definition> diagnosisDefs = new ArrayList<>();
         LOGGER.trace("Diagnosing unassigned shard [{}] due to reason [{}]", shardRouting.shardId(), shardRouting.unassignedInfo());
-        switch (shardRouting.unassignedInfo().getLastAllocationStatus()) {
+        switch (shardRouting.unassignedInfo().lastAllocationStatus()) {
             case NO_VALID_SHARD_COPY -> diagnosisDefs.add(ACTION_RESTORE_FROM_SNAPSHOT);
             case NO_ATTEMPT -> {
-                if (shardRouting.unassignedInfo().isDelayed()) {
+                if (shardRouting.unassignedInfo().delayed()) {
                     diagnosisDefs.add(DIAGNOSIS_WAIT_FOR_OR_FIX_DELAYED_SHARDS);
                 } else {
                     diagnosisDefs.addAll(explainAllocationsAndDiagnoseDeciders(shardRouting, state));
diff --git a/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index fa9636dc89d69..d07d2498d6534 100644
--- a/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -65,7 +65,7 @@ public void processExistingRecoveries(RoutingAllocation allocation, Predicate<Sh
                 }
 
                 // if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
-                if (shard.unassignedInfo() != null && shard.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED) {
+                if (shard.unassignedInfo() != null && shard.unassignedInfo().reason() == UnassignedInfo.Reason.INDEX_CREATED) {
                     continue;
                 }
 
@@ -103,7 +103,7 @@ && canPerformOperationBasedRecovery(primaryStore, shardStores, currentNode) == f
                         );
                         final Set<String> failedNodeIds = shard.unassignedInfo() == null
                             ? Collections.emptySet()
-                            : shard.unassignedInfo().getFailedNodeIds();
+                            : shard.unassignedInfo().failedNodeIds();
                         UnassignedInfo unassignedInfo = new UnassignedInfo(
                             UnassignedInfo.Reason.REALLOCATED_REPLICA,
                             "existing allocation of replica to ["
@@ -138,7 +138,7 @@ private static boolean isResponsibleFor(final ShardRouting shard) {
         return shard.primary() == false // must be a replica
             && shard.unassigned() // must be unassigned
             // if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
-            && shard.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED;
+            && shard.unassignedInfo().reason() != UnassignedInfo.Reason.INDEX_CREATED;
     }
 
     @Override
@@ -234,7 +234,7 @@ public AllocateUnassignedDecision makeAllocationDecision(
                 // we found a match
                 return AllocateUnassignedDecision.yes(nodeWithHighestMatch.node(), null, nodeDecisions, true);
             }
-        } else if (matchingNodes.hasAnyData() == false && unassignedShard.unassignedInfo().isDelayed()) {
+        } else if (matchingNodes.hasAnyData() == false && unassignedShard.unassignedInfo().delayed()) {
             // if we didn't manage to find *any* data (regardless of matching sizes), and the replica is
             // unassigned due to a node leaving, so we delay allocation of this replica to see if the
             // node with the shard copy will rejoin so we can re-use the copy it has
@@ -262,7 +262,7 @@ public static AllocateUnassignedDecision delayedDecision(
             Metadata metadata = allocation.metadata();
             IndexMetadata indexMetadata = metadata.index(unassignedShard.index());
             totalDelayMillis = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexMetadata.getSettings()).getMillis();
-            long remainingDelayNanos = unassignedInfo.getRemainingDelay(
+            long remainingDelayNanos = unassignedInfo.remainingDelay(
                 System.nanoTime(),
                 indexMetadata.getSettings(),
                 metadata.nodeShutdowns()
@@ -357,7 +357,7 @@ private MatchingNodes findMatchingNodes(
             DiscoveryNode discoNode = nodeStoreEntry.getKey();
             if (noMatchFailedNodes
                 && shard.unassignedInfo() != null
-                && shard.unassignedInfo().getFailedNodeIds().contains(discoNode.getId())) {
+                && shard.unassignedInfo().failedNodeIds().contains(discoNode.getId())) {
                 continue;
             }
             TransportNodesListShardStoreMetadata.StoreFilesMetadata storeFilesMetadata = nodeStoreEntry.getValue().storeFilesMetadata();
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 664f9b63dee2a..d9a34fe36c860 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -324,13 +324,13 @@ Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsRe
             table.addCell(commitStats == null ? null : commitStats.getUserData().get(Engine.SYNC_COMMIT_ID));
 
             if (shard.unassignedInfo() != null) {
-                table.addCell(shard.unassignedInfo().getReason());
-                Instant unassignedTime = Instant.ofEpochMilli(shard.unassignedInfo().getUnassignedTimeInMillis());
+                table.addCell(shard.unassignedInfo().reason());
+                Instant unassignedTime = Instant.ofEpochMilli(shard.unassignedInfo().unassignedTimeMillis());
                 table.addCell(UnassignedInfo.DATE_TIME_FORMATTER.format(unassignedTime));
                 table.addCell(
-                    TimeValue.timeValueMillis(Math.max(0, System.currentTimeMillis() - shard.unassignedInfo().getUnassignedTimeInMillis()))
+                    TimeValue.timeValueMillis(Math.max(0, System.currentTimeMillis() - shard.unassignedInfo().unassignedTimeMillis()))
                 );
-                table.addCell(shard.unassignedInfo().getDetails());
+                table.addCell(shard.unassignedInfo().details());
             } else {
                 table.addCell(null);
                 table.addCell(null);
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 74b8a3e12dad5..453d0b3201560 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -795,13 +795,13 @@ public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo)
                     // mark restore entry for this shard as failed when it's due to a file corruption. There is no need wait on retries
                     // to restore this shard on another node if the snapshot files are corrupt. In case where a node just left or crashed,
                     // however, we only want to acknowledge the restore operation once it has been successfully restored on another node.
-                    if (unassignedInfo.getFailure() != null && Lucene.isCorruptionException(unassignedInfo.getFailure().getCause())) {
+                    if (unassignedInfo.failure() != null && Lucene.isCorruptionException(unassignedInfo.failure().getCause())) {
                         changes(recoverySource).put(
                             failedShard.shardId(),
                             new ShardRestoreStatus(
                                 failedShard.currentNodeId(),
                                 RestoreInProgress.State.FAILURE,
-                                unassignedInfo.getFailure().getCause().getMessage()
+                                unassignedInfo.failure().getCause().getMessage()
                             )
                         );
                     }
@@ -829,7 +829,7 @@ public void shardInitialized(ShardRouting unassignedShard, ShardRouting initiali
         public void unassignedInfoUpdated(ShardRouting unassignedShard, UnassignedInfo newUnassignedInfo) {
             RecoverySource recoverySource = unassignedShard.recoverySource();
             if (recoverySource.getType() == RecoverySource.Type.SNAPSHOT) {
-                if (newUnassignedInfo.getLastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO) {
+                if (newUnassignedInfo.lastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO) {
                     String reason = "shard could not be allocated to any of the nodes";
                     changes(recoverySource).put(
                         unassignedShard.shardId(),
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
index f9483bd23f216..eb1a64ef66bbd 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
@@ -103,9 +103,9 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
                     """
                         ,"unassigned_info": {"reason": "%s", "at": "%s", "last_allocation_status": "%s"}
                         """,
-                    shard.unassignedInfo().getReason(),
-                    UnassignedInfo.DATE_TIME_FORMATTER.format(Instant.ofEpochMilli(shard.unassignedInfo().getUnassignedTimeInMillis())),
-                    AllocationDecision.fromAllocationStatus(shard.unassignedInfo().getLastAllocationStatus())
+                    shard.unassignedInfo().reason(),
+                    UnassignedInfo.DATE_TIME_FORMATTER.format(Instant.ofEpochMilli(shard.unassignedInfo().unassignedTimeMillis())),
+                    AllocationDecision.fromAllocationStatus(shard.unassignedInfo().lastAllocationStatus())
                 )
                 : "",
             cae.getCurrentNode().getId(),
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
index a6d380bc7683c..19c268100d4a0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
@@ -162,7 +162,7 @@ public void testClusterStateUpdateTask() {
     private void assertStateAndFailedAllocations(IndexRoutingTable indexRoutingTable, ShardRoutingState state, int failedAllocations) {
         assertThat(indexRoutingTable.size(), equalTo(1));
         assertThat(indexRoutingTable.shard(0).shard(0).state(), equalTo(state));
-        assertThat(indexRoutingTable.shard(0).shard(0).unassignedInfo().getNumFailedAllocations(), equalTo(failedAllocations));
+        assertThat(indexRoutingTable.shard(0).shard(0).unassignedInfo().failedAllocations(), equalTo(failedAllocations));
     }
 
     private ClusterState createInitialClusterState(AllocationService service) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index 05e345bf4b52b..96ff00488a1d2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -559,10 +559,10 @@ private boolean primaryInactiveDueToRecovery(final String indexName, final Clust
                     && primaryShard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE) {
                     return false;
                 }
-                if (primaryShard.unassignedInfo().getNumFailedAllocations() > 0) {
+                if (primaryShard.unassignedInfo().failedAllocations() > 0) {
                     return false;
                 }
-                if (primaryShard.unassignedInfo().getLastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO) {
+                if (primaryShard.unassignedInfo().lastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO) {
                     return false;
                 }
             }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java
index 6e24735eba454..e034971482bcf 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java
@@ -457,7 +457,7 @@ private static void assertIsClosed(final String indexName, final ClusterState cl
             assertThat(
                 RoutingNodesHelper.asStream(shardRoutingTable)
                     .map(ShardRouting::unassignedInfo)
-                    .map(UnassignedInfo::getReason)
+                    .map(UnassignedInfo::reason)
                     .allMatch(info -> info == UnassignedInfo.Reason.INDEX_CLOSED),
                 is(true)
             );
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
index aacf9f803dde0..171fd397d65f3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
@@ -109,7 +109,7 @@ public void testNoDelayedUnassigned() {
             assertThat(unassignedShards.size(), equalTo(0));
         } else {
             assertThat(unassignedShards.size(), equalTo(1));
-            assertThat(unassignedShards.get(0).unassignedInfo().isDelayed(), equalTo(false));
+            assertThat(unassignedShards.get(0).unassignedInfo().delayed(), equalTo(false));
         }
 
         delayedAllocationService.clusterChanged(new ClusterChangedEvent("test", newState, prevState));
@@ -169,7 +169,7 @@ public void testDelayedUnassignedScheduleReroute() throws Exception {
         // make sure the replica is marked as delayed (i.e. not reallocated)
         assertEquals(1, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithDelayedShard));
         ShardRouting delayedShard = stateWithDelayedShard.getRoutingNodes().unassigned().iterator().next();
-        assertEquals(baseTimestampNanos, delayedShard.unassignedInfo().getUnassignedTimeInNanos());
+        assertEquals(baseTimestampNanos, delayedShard.unassignedInfo().unassignedTimeNanos());
 
         // mock ClusterService.submitStateUpdateTask() method
         CountDownLatch latch = new CountDownLatch(1);
@@ -318,8 +318,8 @@ public void testDelayedUnassignedScheduleRerouteAfterDelayedReroute() throws Exc
         final ClusterState stateWithDelayedShards = clusterState;
         assertEquals(2, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithDelayedShards));
         RoutingNodes.UnassignedShards.UnassignedIterator iter = stateWithDelayedShards.getRoutingNodes().unassigned().iterator();
-        assertEquals(baseTimestampNanos, iter.next().unassignedInfo().getUnassignedTimeInNanos());
-        assertEquals(baseTimestampNanos, iter.next().unassignedInfo().getUnassignedTimeInNanos());
+        assertEquals(baseTimestampNanos, iter.next().unassignedInfo().unassignedTimeNanos());
+        assertEquals(baseTimestampNanos, iter.next().unassignedInfo().unassignedTimeNanos());
 
         // mock ClusterService.submitStateUpdateTask() method
         CountDownLatch latch1 = new CountDownLatch(1);
@@ -491,7 +491,7 @@ public void testDelayedUnassignedScheduleRerouteRescheduledOnShorterDelay() {
         // make sure the replica is marked as delayed (i.e. not reallocated)
         assertEquals(1, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithDelayedShard));
         ShardRouting delayedShard = stateWithDelayedShard.getRoutingNodes().unassigned().iterator().next();
-        assertEquals(nodeLeftTimestampNanos, delayedShard.unassignedInfo().getUnassignedTimeInNanos());
+        assertEquals(nodeLeftTimestampNanos, delayedShard.unassignedInfo().unassignedTimeNanos());
 
         assertNull(delayedAllocationService.delayedRerouteTask.get());
         long delayUntilClusterChangeEvent = TimeValue.timeValueNanos(randomInt((int) shorterDelaySetting.nanos() - 1)).nanos();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
index e6466b9237d3a..33695883aebc3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
@@ -401,7 +401,7 @@ public void testEqualsIgnoringVersion() {
                     .withUnassignedInfo(
                         otherRouting.unassignedInfo() == null
                             ? new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test")
-                            : new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, otherRouting.unassignedInfo().getMessage() + "_1")
+                            : new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, otherRouting.unassignedInfo().message() + "_1")
                     )
                     .build();
             }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
index 1d0b01a10da78..eb39d56346eb2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
@@ -137,14 +137,14 @@ public void testSerialization() throws Exception {
         meta.writeTo(out);
         out.close();
 
-        UnassignedInfo read = new UnassignedInfo(out.bytes().streamInput());
-        assertThat(read.getReason(), equalTo(meta.getReason()));
-        assertThat(read.getUnassignedTimeInMillis(), equalTo(meta.getUnassignedTimeInMillis()));
-        assertThat(read.getMessage(), equalTo(meta.getMessage()));
-        assertThat(read.getDetails(), equalTo(meta.getDetails()));
-        assertThat(read.getNumFailedAllocations(), equalTo(meta.getNumFailedAllocations()));
-        assertThat(read.getFailedNodeIds(), equalTo(meta.getFailedNodeIds()));
-        assertThat(read.getLastAllocatedNodeId(), equalTo(meta.getLastAllocatedNodeId()));
+        UnassignedInfo read = UnassignedInfo.fromStreamInput(out.bytes().streamInput());
+        assertThat(read.reason(), equalTo(meta.reason()));
+        assertThat(read.unassignedTimeMillis(), equalTo(meta.unassignedTimeMillis()));
+        assertThat(read.message(), equalTo(meta.message()));
+        assertThat(read.details(), equalTo(meta.details()));
+        assertThat(read.failedAllocations(), equalTo(meta.failedAllocations()));
+        assertThat(read.failedNodeIds(), equalTo(meta.failedNodeIds()));
+        assertThat(read.lastAllocatedNodeId(), equalTo(meta.lastAllocatedNodeId()));
     }
 
     public void testIndexCreated() {
@@ -161,7 +161,7 @@ public void testIndexCreated() {
             .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build())
             .build();
         for (ShardRouting shard : shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED)) {
-            assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.INDEX_CREATED));
+            assertThat(shard.unassignedInfo().reason(), equalTo(UnassignedInfo.Reason.INDEX_CREATED));
         }
     }
 
@@ -181,7 +181,7 @@ public void testClusterRecovered() {
             )
             .build();
         for (ShardRouting shard : shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED)) {
-            assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.CLUSTER_RECOVERED));
+            assertThat(shard.unassignedInfo().reason(), equalTo(UnassignedInfo.Reason.CLUSTER_RECOVERED));
         }
     }
 
@@ -296,8 +296,8 @@ private void assertLastAllocatedNodeIdsAssigned(
             for (int shardCopy = 0; shardCopy < shardRoutingTable.size(); shardCopy++) {
                 final var shard = shardRoutingTable.shard(shardCopy);
                 assertTrue(shard.unassigned());
-                assertThat(shard.unassignedInfo().getReason(), equalTo(expectedUnassignedReason));
-                final var lastAllocatedNodeId = shard.unassignedInfo().getLastAllocatedNodeId();
+                assertThat(shard.unassignedInfo().reason(), equalTo(expectedUnassignedReason));
+                final var lastAllocatedNodeId = shard.unassignedInfo().lastAllocatedNodeId();
                 if (lastAllocatedNodeId == null) {
                     // restoring an index may change the number of shards/replicas so no guarantee that lastAllocatedNodeId is populated
                     assertTrue(shardCountChanged);
@@ -309,7 +309,7 @@ private void assertLastAllocatedNodeIdsAssigned(
             if (shardCountChanged == false) {
                 assertNotNull(previousShardRoutingTable);
                 assertThat(
-                    shardRoutingTable.primaryShard().unassignedInfo().getLastAllocatedNodeId(),
+                    shardRoutingTable.primaryShard().unassignedInfo().lastAllocatedNodeId(),
                     equalTo(previousShardRoutingTable.primaryShard().currentNodeId())
                 );
             }
@@ -335,7 +335,7 @@ public void testIndexReopened() {
             )
             .build();
         for (ShardRouting shard : shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED)) {
-            assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.INDEX_REOPENED));
+            assertThat(shard.unassignedInfo().reason(), equalTo(UnassignedInfo.Reason.INDEX_REOPENED));
         }
     }
 
@@ -366,7 +366,7 @@ public void testNewIndexRestored() {
             )
             .build();
         for (ShardRouting shard : shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED)) {
-            assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.NEW_INDEX_RESTORED));
+            assertThat(shard.unassignedInfo().reason(), equalTo(UnassignedInfo.Reason.NEW_INDEX_RESTORED));
         }
     }
 
@@ -471,7 +471,7 @@ public void testDanglingIndexImported() {
             )
             .build();
         for (ShardRouting shard : shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED)) {
-            assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.DANGLING_INDEX_IMPORTED));
+            assertThat(shard.unassignedInfo().reason(), equalTo(UnassignedInfo.Reason.DANGLING_INDEX_IMPORTED));
         }
     }
 
@@ -501,7 +501,7 @@ public void testReplicaAdded() {
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).size(), equalTo(1));
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo(), notNullValue());
         assertThat(
-            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().getReason(),
+            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().reason(),
             equalTo(UnassignedInfo.Reason.REPLICA_ADDED)
         );
     }
@@ -551,11 +551,11 @@ public void testNodeLeave() {
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).size(), equalTo(1));
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo(), notNullValue());
         assertThat(
-            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().getReason(),
+            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().reason(),
             equalTo(UnassignedInfo.Reason.NODE_LEFT)
         );
         assertThat(
-            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().getUnassignedTimeInMillis(),
+            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().unassignedTimeMillis(),
             greaterThan(0L)
         );
     }
@@ -593,19 +593,19 @@ public void testFailedShard() {
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).size(), equalTo(1));
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo(), notNullValue());
         assertThat(
-            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().getReason(),
+            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().reason(),
             equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED)
         );
         assertThat(
-            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().getMessage(),
+            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().message(),
             equalTo("failed shard on node [" + shardToFail.currentNodeId() + "]: test fail")
         );
         assertThat(
-            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().getDetails(),
+            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().details(),
             equalTo("failed shard on node [" + shardToFail.currentNodeId() + "]: test fail")
         );
         assertThat(
-            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().getUnassignedTimeInMillis(),
+            shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).get(0).unassignedInfo().unassignedTimeMillis(),
             greaterThan(0L)
         );
     }
@@ -768,14 +768,14 @@ private void checkRemainingDelayCalculation(
         final Settings indexSettings = Settings.builder()
             .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), indexLevelTimeoutSetting)
             .build();
-        long delay = unassignedInfo.getRemainingDelay(baseTime, indexSettings, nodeShutdowns);
+        long delay = unassignedInfo.remainingDelay(baseTime, indexSettings, nodeShutdowns);
         assertThat(delay, equalTo(totalDelayNanos));
         long delta1 = randomLongBetween(1, (totalDelayNanos - 1));
-        delay = unassignedInfo.getRemainingDelay(baseTime + delta1, indexSettings, nodeShutdowns);
+        delay = unassignedInfo.remainingDelay(baseTime + delta1, indexSettings, nodeShutdowns);
         assertThat(delay, equalTo(totalDelayNanos - delta1));
-        delay = unassignedInfo.getRemainingDelay(baseTime + totalDelayNanos, indexSettings, nodeShutdowns);
+        delay = unassignedInfo.remainingDelay(baseTime + totalDelayNanos, indexSettings, nodeShutdowns);
         assertThat(delay, equalTo(0L));
-        delay = unassignedInfo.getRemainingDelay(baseTime + totalDelayNanos + randomIntBetween(1, 20), indexSettings, nodeShutdowns);
+        delay = unassignedInfo.remainingDelay(baseTime + totalDelayNanos + randomIntBetween(1, 20), indexSettings, nodeShutdowns);
         assertThat(delay, equalTo(0L));
     }
 
@@ -918,25 +918,25 @@ public void testSummaryContainsImportantFields() {
         var info = randomUnassignedInfo(randomBoolean() ? randomIdentifier() : null);
         var summary = info.shortSummary();
 
-        assertThat("reason", summary, containsString("[reason=" + info.getReason() + ']'));
+        assertThat("reason", summary, containsString("[reason=" + info.reason() + ']'));
         assertThat(
             "delay",
             summary,
-            containsString("at[" + UnassignedInfo.DATE_TIME_FORMATTER.format(Instant.ofEpochMilli(info.getUnassignedTimeInMillis())) + ']')
+            containsString("at[" + UnassignedInfo.DATE_TIME_FORMATTER.format(Instant.ofEpochMilli(info.unassignedTimeMillis())) + ']')
         );
-        if (info.getNumFailedAllocations() > 0) {
-            assertThat("failed_allocations", summary, containsString("failed_attempts[" + info.getNumFailedAllocations() + ']'));
+        if (info.failedAllocations() > 0) {
+            assertThat("failed_allocations", summary, containsString("failed_attempts[" + info.failedAllocations() + ']'));
         }
-        if (info.getFailedNodeIds().isEmpty() == false) {
-            assertThat("failed_nodes", summary, containsString("failed_nodes[" + info.getFailedNodeIds() + ']'));
+        if (info.failedNodeIds().isEmpty() == false) {
+            assertThat("failed_nodes", summary, containsString("failed_nodes[" + info.failedNodeIds() + ']'));
         }
-        assertThat("delayed", summary, containsString("delayed=" + info.isDelayed()));
-        if (info.getLastAllocatedNodeId() != null) {
-            assertThat("last_node", summary, containsString("last_node[" + info.getLastAllocatedNodeId() + ']'));
+        assertThat("delayed", summary, containsString("delayed=" + info.delayed()));
+        if (info.lastAllocatedNodeId() != null) {
+            assertThat("last_node", summary, containsString("last_node[" + info.lastAllocatedNodeId() + ']'));
         }
-        if (info.getMessage() != null) {
-            assertThat("details", summary, containsString("details[" + info.getMessage() + ']'));
+        if (info.message() != null) {
+            assertThat("details", summary, containsString("details[" + info.message() + ']'));
         }
-        assertThat("allocation_status", summary, containsString("allocation_status[" + info.getLastAllocationStatus().value() + ']'));
+        assertThat("allocation_status", summary, containsString("allocation_status[" + info.lastAllocationStatus().value() + ']'));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
index d28c1875100bb..e863aca526da7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
@@ -91,8 +91,8 @@ public void testSingleRetryOnIgnore() {
             routingTable = newState.routingTable();
             assertEquals(routingTable.index("idx").size(), 1);
             assertEquals(routingTable.index("idx").shard(0).shard(0).state(), INITIALIZING);
-            assertEquals(routingTable.index("idx").shard(0).shard(0).unassignedInfo().getNumFailedAllocations(), i + 1);
-            assertThat(routingTable.index("idx").shard(0).shard(0).unassignedInfo().getMessage(), containsString("boom" + i));
+            assertEquals(routingTable.index("idx").shard(0).shard(0).unassignedInfo().failedAllocations(), i + 1);
+            assertThat(routingTable.index("idx").shard(0).shard(0).unassignedInfo().message(), containsString("boom" + i));
         }
         // now we go and check that we are actually stick to unassigned on the next failure
         ClusterState newState = applyShardFailure(clusterState, routingTable.index("idx").shard(0).shard(0), "boom");
@@ -100,9 +100,9 @@ public void testSingleRetryOnIgnore() {
         clusterState = newState;
         routingTable = newState.routingTable();
         assertEquals(routingTable.index("idx").size(), 1);
-        assertEquals(routingTable.index("idx").shard(0).shard(0).unassignedInfo().getNumFailedAllocations(), retries);
+        assertEquals(routingTable.index("idx").shard(0).shard(0).unassignedInfo().failedAllocations(), retries);
         assertEquals(routingTable.index("idx").shard(0).shard(0).state(), UNASSIGNED);
-        assertThat(routingTable.index("idx").shard(0).shard(0).unassignedInfo().getMessage(), containsString("boom"));
+        assertThat(routingTable.index("idx").shard(0).shard(0).unassignedInfo().message(), containsString("boom"));
 
         // manual resetting of retry count
         newState = strategy.reroute(clusterState, new AllocationCommands(), false, true, false, ActionListener.noop()).clusterState();
@@ -112,9 +112,9 @@ public void testSingleRetryOnIgnore() {
 
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         assertEquals(routingTable.index("idx").size(), 1);
-        assertEquals(0, routingTable.index("idx").shard(0).shard(0).unassignedInfo().getNumFailedAllocations());
+        assertEquals(0, routingTable.index("idx").shard(0).shard(0).unassignedInfo().failedAllocations());
         assertEquals(INITIALIZING, routingTable.index("idx").shard(0).shard(0).state());
-        assertThat(routingTable.index("idx").shard(0).shard(0).unassignedInfo().getMessage(), containsString("boom"));
+        assertThat(routingTable.index("idx").shard(0).shard(0).unassignedInfo().message(), containsString("boom"));
 
         // again fail it N-1 times
         for (int i = 0; i < retries - 1; i++) {
@@ -123,9 +123,9 @@ public void testSingleRetryOnIgnore() {
             clusterState = newState;
             routingTable = newState.routingTable();
             assertEquals(routingTable.index("idx").size(), 1);
-            assertEquals(i + 1, routingTable.index("idx").shard(0).shard(0).unassignedInfo().getNumFailedAllocations());
+            assertEquals(i + 1, routingTable.index("idx").shard(0).shard(0).unassignedInfo().failedAllocations());
             assertEquals(INITIALIZING, routingTable.index("idx").shard(0).shard(0).state());
-            assertThat(routingTable.index("idx").shard(0).shard(0).unassignedInfo().getMessage(), containsString("boom"));
+            assertThat(routingTable.index("idx").shard(0).shard(0).unassignedInfo().message(), containsString("boom"));
         }
 
         // now we go and check that we are actually stick to unassigned on the next failure
@@ -134,9 +134,9 @@ public void testSingleRetryOnIgnore() {
         clusterState = newState;
         routingTable = newState.routingTable();
         assertEquals(routingTable.index("idx").size(), 1);
-        assertEquals(retries, routingTable.index("idx").shard(0).shard(0).unassignedInfo().getNumFailedAllocations());
+        assertEquals(retries, routingTable.index("idx").shard(0).shard(0).unassignedInfo().failedAllocations());
         assertEquals(UNASSIGNED, routingTable.index("idx").shard(0).shard(0).state());
-        assertThat(routingTable.index("idx").shard(0).shard(0).unassignedInfo().getMessage(), containsString("boom"));
+        assertThat(routingTable.index("idx").shard(0).shard(0).unassignedInfo().message(), containsString("boom"));
     }
 
     public void testFailedAllocation() {
@@ -152,8 +152,8 @@ public void testFailedAllocation() {
             assertEquals(routingTable.index("idx").size(), 1);
             ShardRouting unassignedPrimary = routingTable.index("idx").shard(0).shard(0);
             assertEquals(unassignedPrimary.state(), INITIALIZING);
-            assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), i + 1);
-            assertThat(unassignedPrimary.unassignedInfo().getMessage(), containsString("boom" + i));
+            assertEquals(unassignedPrimary.unassignedInfo().failedAllocations(), i + 1);
+            assertThat(unassignedPrimary.unassignedInfo().message(), containsString("boom" + i));
             // MaxRetryAllocationDecider#canForceAllocatePrimary should return YES decisions because canAllocate returns YES here
             assertEquals(
                 Decision.Type.YES,
@@ -168,9 +168,9 @@ public void testFailedAllocation() {
             routingTable = newState.routingTable();
             assertEquals(routingTable.index("idx").size(), 1);
             ShardRouting unassignedPrimary = routingTable.index("idx").shard(0).shard(0);
-            assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), retries);
+            assertEquals(unassignedPrimary.unassignedInfo().failedAllocations(), retries);
             assertEquals(unassignedPrimary.state(), UNASSIGNED);
-            assertThat(unassignedPrimary.unassignedInfo().getMessage(), containsString("boom"));
+            assertThat(unassignedPrimary.unassignedInfo().message(), containsString("boom"));
             // MaxRetryAllocationDecider#canForceAllocatePrimary should return a NO decision because canAllocate returns NO here
             final var allocation = newRoutingAllocation(clusterState);
             allocation.debugDecision(true);
@@ -211,9 +211,9 @@ public void testFailedAllocation() {
         // good we are initializing and we are maintaining failure information
         assertEquals(routingTable.index("idx").size(), 1);
         ShardRouting unassignedPrimary = routingTable.index("idx").shard(0).shard(0);
-        assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), retries);
+        assertEquals(unassignedPrimary.unassignedInfo().failedAllocations(), retries);
         assertEquals(unassignedPrimary.state(), INITIALIZING);
-        assertThat(unassignedPrimary.unassignedInfo().getMessage(), containsString("boom"));
+        assertThat(unassignedPrimary.unassignedInfo().message(), containsString("boom"));
         // bumped up the max retry count, so canForceAllocatePrimary should return a YES decision
         assertEquals(
             Decision.Type.YES,
@@ -236,9 +236,9 @@ public void testFailedAllocation() {
         routingTable = newState.routingTable();
         assertEquals(routingTable.index("idx").size(), 1);
         unassignedPrimary = routingTable.index("idx").shard(0).shard(0);
-        assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), 1);
+        assertEquals(unassignedPrimary.unassignedInfo().failedAllocations(), 1);
         assertEquals(unassignedPrimary.state(), UNASSIGNED);
-        assertThat(unassignedPrimary.unassignedInfo().getMessage(), containsString("ZOOOMG"));
+        assertThat(unassignedPrimary.unassignedInfo().message(), containsString("ZOOOMG"));
         // Counter reset, so MaxRetryAllocationDecider#canForceAllocatePrimary should return a YES decision
         assertEquals(
             Decision.Type.YES,
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TrackFailedAllocationNodesTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TrackFailedAllocationNodesTests.java
index 438ec85c4b997..84eead8d51dc2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TrackFailedAllocationNodesTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TrackFailedAllocationNodesTests.java
@@ -59,17 +59,14 @@ public void testTrackFailedNodes() {
                 List.of(new FailedShard(clusterState.routingTable().index("idx").shard(0).shard(0), null, null, randomBoolean())),
                 List.of()
             );
-            assertThat(
-                clusterState.routingTable().index("idx").shard(0).shard(0).unassignedInfo().getFailedNodeIds(),
-                equalTo(failedNodeIds)
-            );
+            assertThat(clusterState.routingTable().index("idx").shard(0).shard(0).unassignedInfo().failedNodeIds(), equalTo(failedNodeIds));
         }
 
         // reroute with retryFailed=true should discard the failedNodes
         assertThat(clusterState.routingTable().index("idx").shard(0).shard(0).state(), equalTo(ShardRoutingState.UNASSIGNED));
         clusterState = allocationService.reroute(clusterState, new AllocationCommands(), false, true, false, ActionListener.noop())
             .clusterState();
-        assertThat(clusterState.routingTable().index("idx").shard(0).shard(0).unassignedInfo().getFailedNodeIds(), empty());
+        assertThat(clusterState.routingTable().index("idx").shard(0).shard(0).unassignedInfo().failedNodeIds(), empty());
 
         // do not track the failed nodes while shard is started
         clusterState = startInitializingShardsAndReroute(allocationService, clusterState);
@@ -79,6 +76,6 @@ public void testTrackFailedNodes() {
             List.of(new FailedShard(clusterState.routingTable().index("idx").shard(0).primaryShard(), null, null, false)),
             List.of()
         );
-        assertThat(clusterState.routingTable().index("idx").shard(0).primaryShard().unassignedInfo().getFailedNodeIds(), empty());
+        assertThat(clusterState.routingTable().index("idx").shard(0).primaryShard().unassignedInfo().failedNodeIds(), empty());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java
index 2fc84c3f32e79..6c3a4157bb4ba 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java
@@ -167,7 +167,7 @@ public void testIgnoresOutOfScopePrimaries() {
                         .replicaShards()
                         .get(0)
                         .unassignedInfo()
-                        .getLastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO ? 1 : 2
+                        .lastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO ? 1 : 2
                 ),
                 new ShardId(index, 1),
                 new ShardAssignment(Set.of("node-0", "node-1"), 2, 0, 0)
@@ -198,7 +198,7 @@ public void testIgnoresOutOfScopeReplicas() {
                     Set.of("node-0"),
                     2,
                     1,
-                    originalReplicaShard.unassignedInfo().getLastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO ? 0 : 1
+                    originalReplicaShard.unassignedInfo().lastAllocationStatus() == UnassignedInfo.AllocationStatus.DECIDERS_NO ? 0 : 1
                 ),
                 new ShardId(index, 1),
                 new ShardAssignment(Set.of("node-0", "node-1"), 2, 0, 0)
@@ -1301,20 +1301,20 @@ private static ShardRouting mutateAllocationStatus(ShardRouting shardRouting) {
             var unassignedInfo = shardRouting.unassignedInfo();
             return shardRouting.updateUnassigned(
                 new UnassignedInfo(
-                    unassignedInfo.getReason(),
-                    unassignedInfo.getMessage(),
-                    unassignedInfo.getFailure(),
-                    unassignedInfo.getNumFailedAllocations(),
-                    unassignedInfo.getUnassignedTimeInNanos(),
-                    unassignedInfo.getUnassignedTimeInMillis(),
-                    unassignedInfo.isDelayed(),
+                    unassignedInfo.reason(),
+                    unassignedInfo.message(),
+                    unassignedInfo.failure(),
+                    unassignedInfo.failedAllocations(),
+                    unassignedInfo.unassignedTimeNanos(),
+                    unassignedInfo.unassignedTimeMillis(),
+                    unassignedInfo.delayed(),
                     randomFrom(
                         UnassignedInfo.AllocationStatus.DECIDERS_NO,
                         UnassignedInfo.AllocationStatus.NO_ATTEMPT,
                         UnassignedInfo.AllocationStatus.DECIDERS_THROTTLED
                     ),
-                    unassignedInfo.getFailedNodeIds(),
-                    unassignedInfo.getLastAllocatedNodeId()
+                    unassignedInfo.failedNodeIds(),
+                    unassignedInfo.lastAllocatedNodeId()
                 ),
                 shardRouting.recoverySource()
             );
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
index 4ce5e78e308b2..0de27aea5b08f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
@@ -132,19 +132,19 @@ public void testFailsNewPrimariesIfNoDataNodes() {
             final var shardRouting = unassigned.next();
             if (shardRouting.primary() && shardRouting.shardId().id() == 1) {
                 final var unassignedInfo = shardRouting.unassignedInfo();
-                assertThat(unassignedInfo.getLastAllocationStatus(), equalTo(UnassignedInfo.AllocationStatus.NO_ATTEMPT));
+                assertThat(unassignedInfo.lastAllocationStatus(), equalTo(UnassignedInfo.AllocationStatus.NO_ATTEMPT));
                 unassigned.updateUnassigned(
                     new UnassignedInfo(
-                        unassignedInfo.getReason(),
-                        unassignedInfo.getMessage(),
-                        unassignedInfo.getFailure(),
-                        unassignedInfo.getNumFailedAllocations(),
-                        unassignedInfo.getUnassignedTimeInNanos(),
-                        unassignedInfo.getUnassignedTimeInMillis(),
-                        unassignedInfo.isDelayed(),
+                        unassignedInfo.reason(),
+                        unassignedInfo.message(),
+                        unassignedInfo.failure(),
+                        unassignedInfo.failedAllocations(),
+                        unassignedInfo.unassignedTimeNanos(),
+                        unassignedInfo.unassignedTimeMillis(),
+                        unassignedInfo.delayed(),
                         UnassignedInfo.AllocationStatus.DECIDERS_THROTTLED,
-                        unassignedInfo.getFailedNodeIds(),
-                        unassignedInfo.getLastAllocatedNodeId()
+                        unassignedInfo.failedNodeIds(),
+                        unassignedInfo.lastAllocatedNodeId()
                     ),
                     shardRouting.recoverySource(),
                     new RoutingChangesObserver.DelegatingRoutingChangesObserver()
@@ -164,7 +164,7 @@ public void testFailsNewPrimariesIfNoDataNodes() {
         for (ShardRouting shardRouting : routingAllocation.routingNodes().unassigned()) {
             assertTrue(shardRouting.toString(), shardRouting.unassigned());
             assertThat(
-                shardRouting.unassignedInfo().getLastAllocationStatus(),
+                shardRouting.unassignedInfo().lastAllocationStatus(),
                 equalTo(
                     shardRouting.primary() && shardRouting.shardId().id() == 1
                         ? UnassignedInfo.AllocationStatus.DECIDERS_THROTTLED
@@ -190,7 +190,7 @@ public void testFailsNewPrimariesIfNoDataNodes() {
         for (ShardRouting shardRouting : routingAllocation.routingNodes().unassigned()) {
             assertTrue(shardRouting.toString(), shardRouting.unassigned());
             assertThat(
-                shardRouting.unassignedInfo().getLastAllocationStatus(),
+                shardRouting.unassignedInfo().lastAllocationStatus(),
                 equalTo(
                     // we only update primaries, and only if currently NO_ATTEMPT
                     shardRouting.primary()
@@ -677,7 +677,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                 .replicaShards()
                 .stream()
                 .allMatch(
-                    shardRouting -> shardRouting.unassignedInfo().getLastAllocationStatus() == UnassignedInfo.AllocationStatus.NO_ATTEMPT
+                    shardRouting -> shardRouting.unassignedInfo().lastAllocationStatus() == UnassignedInfo.AllocationStatus.NO_ATTEMPT
                 )
         );
     }
@@ -724,7 +724,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             nonYesDecision == Decision.NO
                 ? UnassignedInfo.AllocationStatus.DECIDERS_NO
                 : UnassignedInfo.AllocationStatus.DECIDERS_THROTTLED,
-            redState.routingTable().shardRoutingTable("index-0", 0).primaryShard().unassignedInfo().getLastAllocationStatus()
+            redState.routingTable().shardRoutingTable("index-0", 0).primaryShard().unassignedInfo().lastAllocationStatus()
         );
 
         assignPrimary.set(true);
@@ -733,7 +733,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             startInitializingShardsAndReroute(allocationService, redState)
         );
         for (final var shardRouting : yellowState.routingTable().shardRoutingTable("index-0", 0).replicaShards()) {
-            assertEquals(UnassignedInfo.AllocationStatus.NO_ATTEMPT, shardRouting.unassignedInfo().getLastAllocationStatus());
+            assertEquals(UnassignedInfo.AllocationStatus.NO_ATTEMPT, shardRouting.unassignedInfo().lastAllocationStatus());
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java
index 53ac77de6fc88..e5b3393723ab1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java
@@ -100,7 +100,7 @@ public void testGatewayAllocatorStillFetching() {
                 var shardRouting = routingTable.shardRoutingTable("test-index", 0).primaryShard();
                 assertFalse(shardRouting.assignedToNode());
                 assertThat(
-                    shardRouting.unassignedInfo().getLastAllocationStatus(),
+                    shardRouting.unassignedInfo().lastAllocationStatus(),
                     equalTo(UnassignedInfo.AllocationStatus.FETCHING_SHARD_DATA)
                 );
             }
@@ -111,7 +111,7 @@ public void testGatewayAllocatorDoesNothing() {
         testAllocate((allocation, unassignedAllocationHandler) -> {}, routingTable -> {
             var shardRouting = routingTable.shardRoutingTable("test-index", 0).primaryShard();
             assertTrue(shardRouting.assignedToNode());// assigned by a followup reconciliation
-            assertThat(shardRouting.unassignedInfo().getLastAllocationStatus(), equalTo(UnassignedInfo.AllocationStatus.NO_ATTEMPT));
+            assertThat(shardRouting.unassignedInfo().lastAllocationStatus(), equalTo(UnassignedInfo.AllocationStatus.NO_ATTEMPT));
         });
     }
 
@@ -328,7 +328,7 @@ protected long currentNanoTime() {
             var unassigned = reconciledState.getRoutingNodes().unassigned();
             assertThat(unassigned.size(), equalTo(1));
             var unassignedShard = unassigned.iterator().next();
-            assertThat(unassignedShard.unassignedInfo().isDelayed(), equalTo(true));
+            assertThat(unassignedShard.unassignedInfo().delayed(), equalTo(true));
 
         } finally {
             clusterService.close();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index 716e7c80a6cde..d5cf73cacb782 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -1190,13 +1190,13 @@ private void doTestDiskThresholdWithSnapshotShardSizes(boolean testMaxHeadroom)
         assertThat(
             shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).stream()
                 .map(ShardRouting::unassignedInfo)
-                .allMatch(unassignedInfo -> Reason.NEW_INDEX_RESTORED.equals(unassignedInfo.getReason())),
+                .allMatch(unassignedInfo -> Reason.NEW_INDEX_RESTORED.equals(unassignedInfo.reason())),
             is(true)
         );
         assertThat(
             shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).stream()
                 .map(ShardRouting::unassignedInfo)
-                .allMatch(unassignedInfo -> AllocationStatus.NO_ATTEMPT.equals(unassignedInfo.getLastAllocationStatus())),
+                .allMatch(unassignedInfo -> AllocationStatus.NO_ATTEMPT.equals(unassignedInfo.lastAllocationStatus())),
             is(true)
         );
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).size(), equalTo(1));
@@ -1218,7 +1218,7 @@ private void doTestDiskThresholdWithSnapshotShardSizes(boolean testMaxHeadroom)
         assertThat(
             shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).stream()
                 .map(ShardRouting::unassignedInfo)
-                .allMatch(unassignedInfo -> AllocationStatus.FETCHING_SHARD_DATA.equals(unassignedInfo.getLastAllocationStatus())),
+                .allMatch(unassignedInfo -> AllocationStatus.FETCHING_SHARD_DATA.equals(unassignedInfo.lastAllocationStatus())),
             is(true)
         );
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).size(), equalTo(1));
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
index ea156ee48a656..ab14345cb53c4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
@@ -111,16 +111,16 @@ public void testCanAllocatePrimaryExistingInRestoreInProgress() {
 
             UnassignedInfo currentInfo = primary.unassignedInfo();
             UnassignedInfo newInfo = new UnassignedInfo(
-                currentInfo.getReason(),
-                currentInfo.getMessage(),
+                currentInfo.reason(),
+                currentInfo.message(),
                 new IOException("i/o failure"),
-                currentInfo.getNumFailedAllocations(),
-                currentInfo.getUnassignedTimeInNanos(),
-                currentInfo.getUnassignedTimeInMillis(),
-                currentInfo.isDelayed(),
-                currentInfo.getLastAllocationStatus(),
-                currentInfo.getFailedNodeIds(),
-                currentInfo.getLastAllocatedNodeId()
+                currentInfo.failedAllocations(),
+                currentInfo.unassignedTimeNanos(),
+                currentInfo.unassignedTimeMillis(),
+                currentInfo.delayed(),
+                currentInfo.lastAllocationStatus(),
+                currentInfo.failedNodeIds(),
+                currentInfo.lastAllocatedNodeId()
             );
             primary = primary.updateUnassigned(newInfo, primary.recoverySource());
 
diff --git a/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
index f6b310abac770..a74a00792d701 100644
--- a/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
@@ -286,7 +286,7 @@ public void testDontAllocateOnNoOrThrottleForceAllocationDecision() {
         List<ShardRouting> ignored = allocation.routingNodes().unassigned().ignored();
         assertEquals(ignored.size(), 1);
         assertEquals(
-            ignored.get(0).unassignedInfo().getLastAllocationStatus(),
+            ignored.get(0).unassignedInfo().lastAllocationStatus(),
             forceDecisionNo ? AllocationStatus.DECIDERS_NO : AllocationStatus.DECIDERS_THROTTLED
         );
         assertTrue(shardsWithState(allocation.routingNodes(), ShardRoutingState.INITIALIZING).isEmpty());
@@ -314,7 +314,7 @@ public void testDontForceAllocateOnThrottleDecision() {
         assertThat(allocation.routingNodesChanged(), equalTo(true));
         List<ShardRouting> ignored = allocation.routingNodes().unassigned().ignored();
         assertEquals(ignored.size(), 1);
-        assertEquals(ignored.get(0).unassignedInfo().getLastAllocationStatus(), AllocationStatus.DECIDERS_THROTTLED);
+        assertEquals(ignored.get(0).unassignedInfo().lastAllocationStatus(), AllocationStatus.DECIDERS_THROTTLED);
         assertTrue(shardsWithState(allocation.routingNodes(), ShardRoutingState.INITIALIZING).isEmpty());
     }
 
@@ -454,7 +454,7 @@ public void testRestoreDoesNotAssignIfShardSizeNotAvailable() {
         assertThat(allocation.routingNodesChanged(), equalTo(true));
         assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(false));
         ShardRouting ignoredRouting = allocation.routingNodes().unassigned().ignored().get(0);
-        assertThat(ignoredRouting.unassignedInfo().getLastAllocationStatus(), equalTo(AllocationStatus.FETCHING_SHARD_DATA));
+        assertThat(ignoredRouting.unassignedInfo().lastAllocationStatus(), equalTo(AllocationStatus.FETCHING_SHARD_DATA));
         assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index e1cba6f1746e4..9582037975318 100644
--- a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -254,8 +254,8 @@ private void runNoopRetentionLeaseTest(boolean isRelevantShard) {
             List<ShardRouting> unassignedShards = shardsWithState(allocation.routingNodes(), ShardRoutingState.UNASSIGNED);
             assertThat(unassignedShards, hasSize(1));
             assertThat(unassignedShards.get(0).shardId(), equalTo(shardId));
-            assertThat(unassignedShards.get(0).unassignedInfo().getNumFailedAllocations(), equalTo(0));
-            assertThat(unassignedShards.get(0).unassignedInfo().getFailedNodeIds(), equalTo(failedNodeIds));
+            assertThat(unassignedShards.get(0).unassignedInfo().failedAllocations(), equalTo(0));
+            assertThat(unassignedShards.get(0).unassignedInfo().failedNodeIds(), equalTo(failedNodeIds));
         } else {
             assertThat(allocation.routingNodesChanged(), equalTo(false));
             assertThat(shardsWithState(allocation.routingNodes(), ShardRoutingState.UNASSIGNED).size(), equalTo(0));
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index fa700dc5d78f7..697b40671ee8b 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -1035,7 +1035,7 @@ public void run() {
                             .routingTable()
                             .shardRoutingTable(shardToRelocate.shardId())
                             .primaryShard();
-                        if (shardRouting.unassigned() && shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.NODE_LEFT) {
+                        if (shardRouting.unassigned() && shardRouting.unassignedInfo().reason() == UnassignedInfo.Reason.NODE_LEFT) {
                             if (masterNodeCount > 1) {
                                 scheduleNow(() -> testClusterNodes.stopNode(masterNode));
                             }
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
index 7848f0ef4a625..f3fac694f9980 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
@@ -423,10 +423,10 @@ public void allocateUnassigned(
             RoutingAllocation allocation,
             UnassignedAllocationHandler unassignedAllocationHandler
         ) {
-            if (shardRouting.primary() || shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED) {
+            if (shardRouting.primary() || shardRouting.unassignedInfo().reason() == UnassignedInfo.Reason.INDEX_CREATED) {
                 return;
             }
-            if (shardRouting.unassignedInfo().isDelayed()) {
+            if (shardRouting.unassignedInfo().delayed()) {
                 unassignedAllocationHandler.removeAndIgnore(UnassignedInfo.AllocationStatus.DELAYED_ALLOCATION, allocation.changes());
             }
         }
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java
index dff3ff935595f..90bbc29a11b41 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java
@@ -553,7 +553,7 @@ public void testCcrRepositoryFetchesSnapshotShardSizeFromIndexShardStoreStats()
             if (RestoreInProgress.get(event.state()).isEmpty() == false && event.state().routingTable().hasIndex(followerIndex)) {
                 final IndexRoutingTable indexRoutingTable = event.state().routingTable().index(followerIndex);
                 for (ShardRouting shardRouting : indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED)) {
-                    if (shardRouting.unassignedInfo().getLastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA) {
+                    if (shardRouting.unassignedInfo().lastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA) {
                         try {
                             assertBusy(() -> {
                                 final Long snapshotShardSize = snapshotsInfoService.snapshotShardSizes().getShardSize(shardRouting);
@@ -644,7 +644,7 @@ public void testCcrRepositoryFailsToFetchSnapshotShardSizes() throws Exception {
                         assertBusy(() -> {
                             List<Long> sizes = indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED)
                                 .stream()
-                                .filter(shard -> shard.unassignedInfo().getLastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA)
+                                .filter(shard -> shard.unassignedInfo().lastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA)
                                 .sorted(Comparator.comparingInt(ShardRouting::getId))
                                 .map(shard -> snapshotsInfoService.snapshotShardSizes().getShardSize(shard))
                                 .filter(Objects::nonNull)
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java
index ffedcb8f9ebd3..ee19fc07e45cb 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java
@@ -96,7 +96,7 @@ public Settings onNodeStopped(String nodeName) {
                 for (RoutingNode routingNode : event.state().getRoutingNodes()) {
                     for (ShardRouting shardRouting : routingNode) {
                         if (shardRouting.unassignedInfo() != null) {
-                            unassignedReasons.add(shardRouting.unassignedInfo().getReason());
+                            unassignedReasons.add(shardRouting.unassignedInfo().reason());
                         }
                     }
                 }
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocator.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocator.java
index ee018578ce143..b05f7e4844908 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocator.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocator.java
@@ -332,8 +332,8 @@ private AllocateUnassignedDecision decideAllocation(RoutingAllocation allocation
     }
 
     private static boolean isDelayedDueToNodeRestart(RoutingAllocation allocation, ShardRouting shardRouting) {
-        if (shardRouting.unassignedInfo().isDelayed()) {
-            String lastAllocatedNodeId = shardRouting.unassignedInfo().getLastAllocatedNodeId();
+        if (shardRouting.unassignedInfo().delayed()) {
+            String lastAllocatedNodeId = shardRouting.unassignedInfo().lastAllocatedNodeId();
             if (lastAllocatedNodeId != null) {
                 return allocation.metadata().nodeShutdowns().contains(lastAllocatedNodeId, SingleNodeShutdownMetadata.Type.RESTART);
             }
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java
index 9e8c54ba594ea..69043c606ef15 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java
@@ -219,7 +219,7 @@ static ShutdownShardMigrationStatus shardMigrationStatus(
             .unassigned()
             .stream()
             .peek(s -> cancellableTask.ensureNotCancelled())
-            .filter(s -> Objects.equals(s.unassignedInfo().getLastAllocatedNodeId(), nodeId))
+            .filter(s -> Objects.equals(s.unassignedInfo().lastAllocatedNodeId(), nodeId))
             .filter(s -> s.primary() || hasShardCopyOnAnotherNode(currentState, s, shuttingDownNodes) == false)
             .toList();
 

From 4a1d7426d7027930faf117a2c04df507a9b941db Mon Sep 17 00:00:00 2001
From: Panagiotis Bailis <pmpailis@gmail.com>
Date: Thu, 6 Jun 2024 11:20:53 +0300
Subject: [PATCH 189/208] Adding RankFeature implementation  (#108538)

---
 docs/changelog/108538.yaml                    |    5 +
 .../search/rank/FieldBasedRerankerIT.java     |  811 ++++++++++++
 server/src/main/java/module-info.java         |    1 +
 .../org/elasticsearch/TransportVersions.java  |    2 +-
 ...ngleCoordinatorSearchProgressListener.java |   18 +
 .../action/search/FetchSearchPhase.java       |   68 +-
 .../action/search/RankFeaturePhase.java       |  175 ++-
 .../action/search/SearchPhase.java            |   34 +
 .../action/search/SearchPhaseController.java  |    4 +-
 .../action/search/SearchProgressListener.java |   32 +
 .../action/search/SearchRequest.java          |    2 +-
 .../search/SearchTransportAPMMetrics.java     |    1 +
 .../action/search/SearchTransportService.java |   30 +
 .../elasticsearch/node/NodeConstruction.java  |    1 +
 .../node/NodeServiceProvider.java             |    3 +
 .../search/DefaultSearchContext.java          |   13 +
 .../elasticsearch/search/SearchModule.java    |    5 +
 .../search/SearchPhaseResult.java             |   16 +
 .../elasticsearch/search/SearchService.java   |   39 +
 .../search/fetch/FetchPhase.java              |    1 -
 .../internal/FilteredSearchContext.java       |   11 +
 .../search/internal/SearchContext.java        |    5 +
 .../search/query/QueryPhase.java              |   55 +-
 .../search/rank/RankBuilder.java              |   23 +-
 .../search/rank/RankSearchContext.java        |   19 +-
 ...ankFeaturePhaseRankCoordinatorContext.java |   96 ++
 .../RankFeaturePhaseRankShardContext.java     |   39 +
 .../search/rank/feature/RankFeatureDoc.java   |   54 +
 .../rank/feature/RankFeatureResult.java       |   70 +
 .../rank/feature/RankFeatureShardPhase.java   |   99 ++
 .../rank/feature/RankFeatureShardRequest.java |  101 ++
 .../rank/feature/RankFeatureShardResult.java  |   68 +
 .../action/search/RankFeaturePhaseTests.java  | 1170 +++++++++++++++++
 .../search/DefaultSearchContextTests.java     |    4 +-
 .../search/SearchServiceTests.java            |  753 ++++++++++-
 .../rank/RankFeatureShardPhaseTests.java      |  409 ++++++
 .../snapshots/SnapshotResiliencyTests.java    |    2 +
 .../java/org/elasticsearch/node/MockNode.java |    4 +
 .../search/MockSearchService.java             |    3 +
 .../search/rank/TestRankBuilder.java          |   19 +-
 .../elasticsearch/test/TestSearchContext.java |   11 +
 .../hamcrest/ElasticsearchAssertions.java     |    4 +
 .../xpack/search/AsyncSearchTask.java         |   14 +
 .../xpack/rank/rrf/RRFRankBuilder.java        |   19 +-
 .../xpack/rank/rrf/RRFRetrieverBuilder.java   |    2 +-
 .../security/authz/PreAuthorizationUtils.java |    1 +
 46 files changed, 4207 insertions(+), 109 deletions(-)
 create mode 100644 docs/changelog/108538.yaml
 create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankCoordinatorContext.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankShardContext.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureResult.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardRequest.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardResult.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java

diff --git a/docs/changelog/108538.yaml b/docs/changelog/108538.yaml
new file mode 100644
index 0000000000000..10ae49f0c1670
--- /dev/null
+++ b/docs/changelog/108538.yaml
@@ -0,0 +1,5 @@
+pr: 108538
+summary: Adding RankFeature search phase implementation
+area: Search
+type: feature
+issues: []
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java
new file mode 100644
index 0000000000000..a4e2fda0fd3c9
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java
@@ -0,0 +1,811 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchPhaseController;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
+import org.elasticsearch.search.rank.feature.RankFeatureDoc;
+import org.elasticsearch.search.rank.feature.RankFeatureShardResult;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasRank;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(minNumDataNodes = 3)
+public class FieldBasedRerankerIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(FieldBasedRerankerPlugin.class);
+    }
+
+    public void testFieldBasedReranker() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        createIndex(indexName);
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E")
+        );
+
+        assertNoFailuresAndResponse(
+            prepareSearch().setQuery(
+                boolQuery().should(constantScoreQuery(matchQuery(searchField, "A")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "B")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "C")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "D")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "E")).boost(randomFloat()))
+            )
+                .setRankBuilder(new FieldBasedRankBuilder(rankWindowSize, rankFeatureField))
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(10),
+            response -> {
+                assertHitCount(response, 5L);
+                int rank = 1;
+                for (SearchHit searchHit : response.getHits().getHits()) {
+                    assertThat(searchHit, hasId(String.valueOf(5 - (rank - 1))));
+                    assertEquals(searchHit.getScore(), (0.5f - ((rank - 1) * 0.1f)), 1e-5f);
+                    assertThat(searchHit, hasRank(rank));
+                    assertNotNull(searchHit.getFields().get(searchField));
+                    rank++;
+                }
+            }
+        );
+        assertNoOpenContext(indexName);
+    }
+
+    public void testFieldBasedRerankerPagination() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        createIndex(indexName);
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E")
+        );
+
+        assertResponse(
+            prepareSearch().setQuery(
+                boolQuery().should(constantScoreQuery(matchQuery(searchField, "A")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "B")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "C")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "D")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "E")).boost(randomFloat()))
+            )
+                .setRankBuilder(new FieldBasedRankBuilder(rankWindowSize, rankFeatureField))
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(2)
+                .setFrom(2),
+            response -> {
+                assertHitCount(response, 5L);
+                int rank = 3;
+                for (SearchHit searchHit : response.getHits().getHits()) {
+                    assertThat(searchHit, hasId(String.valueOf(5 - (rank - 1))));
+                    assertEquals(searchHit.getScore(), (0.5f - ((rank - 1) * 0.1f)), 1e-5f);
+                    assertThat(searchHit, hasRank(rank));
+                    assertNotNull(searchHit.getFields().get(searchField));
+                    rank++;
+                }
+            }
+        );
+        assertNoOpenContext(indexName);
+    }
+
+    public void testFieldBasedRerankerPaginationOutsideOfBounds() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        createIndex(indexName);
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E")
+        );
+
+        assertNoFailuresAndResponse(
+            prepareSearch().setQuery(
+                boolQuery().should(constantScoreQuery(matchQuery(searchField, "A")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "B")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "C")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "D")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "E")).boost(randomFloat()))
+            )
+                .setRankBuilder(new FieldBasedRankBuilder(rankWindowSize, rankFeatureField))
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(2)
+                .setFrom(10),
+            response -> {
+                assertHitCount(response, 5L);
+                assertEquals(0, response.getHits().getHits().length);
+            }
+        );
+        assertNoOpenContext(indexName);
+    }
+
+    public void testNotAllShardsArePresentInFetchPhase() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 10).build());
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A").setRouting("A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B").setRouting("B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C").setRouting("C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D").setRouting("C"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E").setRouting("C")
+        );
+
+        assertNoFailuresAndResponse(
+            prepareSearch().setQuery(
+                boolQuery().should(constantScoreQuery(matchQuery(searchField, "A")).boost(0.1f))
+                    .should(constantScoreQuery(matchQuery(searchField, "C")).boost(0.3f))
+                    .should(constantScoreQuery(matchQuery(searchField, "D")).boost(0.3f))
+                    .should(constantScoreQuery(matchQuery(searchField, "E")).boost(0.3f))
+            )
+                .setRankBuilder(new FieldBasedRankBuilder(rankWindowSize, rankFeatureField))
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(2),
+            response -> {
+                assertHitCount(response, 4L);
+                assertEquals(2, response.getHits().getHits().length);
+                int rank = 1;
+                for (SearchHit searchHit : response.getHits().getHits()) {
+                    assertThat(searchHit, hasId(String.valueOf(5 - (rank - 1))));
+                    assertEquals(searchHit.getScore(), (0.5f - ((rank - 1) * 0.1f)), 1e-5f);
+                    assertThat(searchHit, hasRank(rank));
+                    assertNotNull(searchHit.getFields().get(searchField));
+                    rank++;
+                }
+            }
+        );
+        assertNoOpenContext(indexName);
+    }
+
+    public void testFieldBasedRerankerNoMatchingDocs() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        createIndex(indexName);
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E")
+        );
+
+        assertNoFailuresAndResponse(
+            prepareSearch().setQuery(boolQuery().should(constantScoreQuery(matchQuery(searchField, "F")).boost(randomFloat())))
+                .setRankBuilder(new FieldBasedRankBuilder(rankWindowSize, rankFeatureField))
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(10),
+            response -> {
+                assertHitCount(response, 0L);
+            }
+        );
+        assertNoOpenContext(indexName);
+    }
+
+    public void testQueryPhaseShardThrowingRankBuilderAllContextsAreClosedAllShardsFail() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        // this test is irrespective of the number of shards, as we will always reach QueryPhaseRankShardContext#combineQueryPhaseResults
+        // even with no results. So, when we get back to the coordinator, all shards will have failed, and the whole response
+        // will be marked as a failure
+        createIndex(indexName);
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E")
+        );
+
+        expectThrows(
+            SearchPhaseExecutionException.class,
+            () -> prepareSearch().setQuery(
+                boolQuery().should(constantScoreQuery(matchQuery(searchField, "A")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "B")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "C")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "D")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "E")).boost(randomFloat()))
+            )
+                .setRankBuilder(
+                    new ThrowingRankBuilder(
+                        rankWindowSize,
+                        rankFeatureField,
+                        ThrowingRankBuilder.ThrowingRankBuilderType.THROWING_QUERY_PHASE_SHARD_CONTEXT.name()
+                    )
+                )
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(10)
+                .get()
+        );
+        assertNoOpenContext(indexName);
+    }
+
+    public void testQueryPhaseCoordinatorThrowingRankBuilderAllContextsAreClosedAllShardsFail() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        createIndex(indexName);
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E")
+        );
+
+        // when we throw on the coordinator, the onPhaseFailure handler will be invoked, which in turn will mark the whole
+        // search request as a failure (i.e. no partial results)
+        expectThrows(
+            SearchPhaseExecutionException.class,
+            () -> prepareSearch().setQuery(
+                boolQuery().should(constantScoreQuery(matchQuery(searchField, "A")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "B")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "C")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "D")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "E")).boost(randomFloat()))
+            )
+                .setRankBuilder(
+                    new ThrowingRankBuilder(
+                        rankWindowSize,
+                        rankFeatureField,
+                        ThrowingRankBuilder.ThrowingRankBuilderType.THROWING_QUERY_PHASE_COORDINATOR_CONTEXT.name()
+                    )
+                )
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(10)
+                .get()
+        );
+        assertNoOpenContext(indexName);
+    }
+
+    public void testRankFeaturePhaseShardThrowingRankBuilderAllContextsAreClosedPartialFailures() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 10).build());
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E")
+        );
+
+        // we have 10 shards and 5 documents, so when the exception is thrown we know that not all shards will report failures
+        assertResponse(
+            prepareSearch().setQuery(
+                boolQuery().should(constantScoreQuery(matchQuery(searchField, "A")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "B")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "C")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "D")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "E")).boost(randomFloat()))
+            )
+                .setRankBuilder(
+                    new ThrowingRankBuilder(
+                        rankWindowSize,
+                        rankFeatureField,
+                        ThrowingRankBuilder.ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_SHARD_CONTEXT.name()
+                    )
+                )
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(10),
+            response -> {
+                assertTrue(response.getFailedShards() > 0);
+                assertTrue(
+                    Arrays.stream(response.getShardFailures())
+                        .allMatch(failure -> failure.getCause().getMessage().contains("rfs - simulated failure"))
+                );
+                assertHitCount(response, 5);
+                assertTrue(response.getHits().getHits().length == 0);
+            }
+        );
+        assertNoOpenContext(indexName);
+    }
+
+    public void testRankFeaturePhaseShardThrowingRankBuilderAllContextsAreClosedAllShardsFail() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        // we have 1 shard and 5 documents, so when the exception is thrown we know that all shards will have failed
+        createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build());
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E")
+        );
+
+        expectThrows(
+            SearchPhaseExecutionException.class,
+            () -> prepareSearch().setQuery(
+                boolQuery().should(constantScoreQuery(matchQuery(searchField, "A")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "B")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "C")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "D")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "E")).boost(randomFloat()))
+            )
+                .setRankBuilder(
+                    new ThrowingRankBuilder(
+                        rankWindowSize,
+                        rankFeatureField,
+                        ThrowingRankBuilder.ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_SHARD_CONTEXT.name()
+                    )
+                )
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(10)
+                .get()
+        );
+        assertNoOpenContext(indexName);
+    }
+
+    public void testRankFeaturePhaseCoordinatorThrowingRankBuilderAllContextsAreClosedAllShardsFail() throws Exception {
+        final String indexName = "test_index";
+        final String rankFeatureField = "rankFeatureField";
+        final String searchField = "searchField";
+        final int rankWindowSize = 10;
+
+        createIndex(indexName);
+        indexRandom(
+            true,
+            prepareIndex(indexName).setId("1").setSource(rankFeatureField, 0.1, searchField, "A"),
+            prepareIndex(indexName).setId("2").setSource(rankFeatureField, 0.2, searchField, "B"),
+            prepareIndex(indexName).setId("3").setSource(rankFeatureField, 0.3, searchField, "C"),
+            prepareIndex(indexName).setId("4").setSource(rankFeatureField, 0.4, searchField, "D"),
+            prepareIndex(indexName).setId("5").setSource(rankFeatureField, 0.5, searchField, "E")
+        );
+
+        expectThrows(
+            SearchPhaseExecutionException.class,
+            () -> prepareSearch().setQuery(
+                boolQuery().should(constantScoreQuery(matchQuery(searchField, "A")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "B")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "C")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "D")).boost(randomFloat()))
+                    .should(constantScoreQuery(matchQuery(searchField, "E")).boost(randomFloat()))
+            )
+                .setRankBuilder(
+                    new ThrowingRankBuilder(
+                        rankWindowSize,
+                        rankFeatureField,
+                        ThrowingRankBuilder.ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_COORDINATOR_CONTEXT.name()
+                    )
+                )
+                .addFetchField(searchField)
+                .setTrackTotalHits(true)
+                .setAllowPartialSearchResults(true)
+                .setSize(10)
+                .get()
+        );
+        assertNoOpenContext(indexName);
+    }
+
+    private void assertNoOpenContext(final String indexName) throws Exception {
+        assertBusy(
+            () -> assertThat(indicesAdmin().prepareStats(indexName).get().getTotal().getSearch().getOpenContexts(), equalTo(0L)),
+            1,
+            TimeUnit.SECONDS
+        );
+    }
+
+    public static class FieldBasedRankBuilder extends RankBuilder {
+
+        public static final ParseField FIELD_FIELD = new ParseField("field");
+        static final ConstructingObjectParser<FieldBasedRankBuilder, Void> PARSER = new ConstructingObjectParser<>(
+            "field-based-rank",
+            args -> {
+                int rankWindowSize = args[0] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[0];
+                String field = (String) args[1];
+                if (field == null || field.isEmpty()) {
+                    throw new IllegalArgumentException("Field cannot be null or empty");
+                }
+                return new FieldBasedRankBuilder(rankWindowSize, field);
+            }
+        );
+
+        static {
+            PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD);
+            PARSER.declareString(constructorArg(), FIELD_FIELD);
+        }
+
+        protected final String field;
+
+        public static FieldBasedRankBuilder fromXContent(XContentParser parser) throws IOException {
+            return PARSER.parse(parser, null);
+        }
+
+        public FieldBasedRankBuilder(final int rankWindowSize, final String field) {
+            super(rankWindowSize);
+            this.field = field;
+        }
+
+        public FieldBasedRankBuilder(StreamInput in) throws IOException {
+            super(in);
+            this.field = in.readString();
+        }
+
+        @Override
+        protected void doWriteTo(StreamOutput out) throws IOException {
+            out.writeString(field);
+        }
+
+        @Override
+        protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(FIELD_FIELD.getPreferredName(), field);
+        }
+
+        @Override
+        public boolean isCompoundBuilder() {
+            return false;
+        }
+
+        @Override
+        public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
+            return new QueryPhaseRankShardContext(queries, rankWindowSize()) {
+                @Override
+                public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
+                    Map<Integer, RankFeatureDoc> rankDocs = new HashMap<>();
+                    rankResults.forEach(topDocs -> {
+                        for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+                            rankDocs.compute(scoreDoc.doc, (key, value) -> {
+                                if (value == null) {
+                                    return new RankFeatureDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex);
+                                } else {
+                                    value.score = Math.max(scoreDoc.score, rankDocs.get(scoreDoc.doc).score);
+                                    return value;
+                                }
+                            });
+                        }
+                    });
+                    RankFeatureDoc[] sortedResults = rankDocs.values().toArray(RankFeatureDoc[]::new);
+                    Arrays.sort(sortedResults, (o1, o2) -> Float.compare(o2.score, o1.score));
+                    return new RankFeatureShardResult(sortedResults);
+                }
+            };
+        }
+
+        @Override
+        public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) {
+            return new QueryPhaseRankCoordinatorContext(rankWindowSize()) {
+                @Override
+                public ScoreDoc[] rankQueryPhaseResults(
+                    List<QuerySearchResult> querySearchResults,
+                    SearchPhaseController.TopDocsStats topDocStats
+                ) {
+                    List<RankFeatureDoc> rankDocs = new ArrayList<>();
+                    for (int i = 0; i < querySearchResults.size(); i++) {
+                        QuerySearchResult querySearchResult = querySearchResults.get(i);
+                        RankFeatureShardResult shardResult = (RankFeatureShardResult) querySearchResult.getRankShardResult();
+                        for (RankFeatureDoc frd : shardResult.rankFeatureDocs) {
+                            frd.shardIndex = i;
+                            rankDocs.add(frd);
+                        }
+                    }
+                    // no support for sort field atm
+                    // should pass needed info to make use of org.elasticsearch.action.search.SearchPhaseController.sortDocs?
+                    rankDocs.sort(Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed());
+                    RankFeatureDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(RankFeatureDoc[]::new);
+
+                    assert topDocStats.fetchHits == 0;
+                    topDocStats.fetchHits = topResults.length;
+
+                    return topResults;
+                }
+            };
+        }
+
+        @Override
+        public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+            return new RankFeaturePhaseRankShardContext(field) {
+                @Override
+                public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) {
+                    try {
+                        RankFeatureDoc[] rankFeatureDocs = new RankFeatureDoc[hits.getHits().length];
+                        for (int i = 0; i < hits.getHits().length; i++) {
+                            rankFeatureDocs[i] = new RankFeatureDoc(hits.getHits()[i].docId(), hits.getHits()[i].getScore(), shardId);
+                            rankFeatureDocs[i].featureData(hits.getHits()[i].field(field).getValue().toString());
+                        }
+                        return new RankFeatureShardResult(rankFeatureDocs);
+                    } catch (Exception ex) {
+                        throw ex;
+                    }
+                }
+            };
+        }
+
+        @Override
+        public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from) {
+            return new RankFeaturePhaseRankCoordinatorContext(size, from, rankWindowSize()) {
+                @Override
+                protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
+                    float[] scores = new float[featureDocs.length];
+                    for (int i = 0; i < featureDocs.length; i++) {
+                        scores[i] = Float.parseFloat(featureDocs[i].featureData);
+                    }
+                    scoreListener.onResponse(scores);
+                }
+            };
+        }
+
+        @Override
+        protected boolean doEquals(RankBuilder other) {
+            return other instanceof FieldBasedRankBuilder && Objects.equals(field, ((FieldBasedRankBuilder) other).field);
+        }
+
+        @Override
+        protected int doHashCode() {
+            return Objects.hash(field);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return "field-based-rank";
+        }
+
+        @Override
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersions.RANK_FEATURE_PHASE_ADDED;
+        }
+    }
+
+    public static class ThrowingRankBuilder extends FieldBasedRankBuilder {
+
+        public enum ThrowingRankBuilderType {
+            THROWING_QUERY_PHASE_SHARD_CONTEXT,
+            THROWING_QUERY_PHASE_COORDINATOR_CONTEXT,
+            THROWING_RANK_FEATURE_PHASE_SHARD_CONTEXT,
+            THROWING_RANK_FEATURE_PHASE_COORDINATOR_CONTEXT;
+        }
+
+        protected final ThrowingRankBuilderType throwingRankBuilderType;
+
+        public static final ParseField FIELD_FIELD = new ParseField("field");
+        public static final ParseField THROWING_TYPE_FIELD = new ParseField("throwing-type");
+        static final ConstructingObjectParser<ThrowingRankBuilder, Void> PARSER = new ConstructingObjectParser<>("throwing-rank", args -> {
+            int rankWindowSize = args[0] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[0];
+            String field = (String) args[1];
+            if (field == null || field.isEmpty()) {
+                throw new IllegalArgumentException("Field cannot be null or empty");
+            }
+            String throwingType = (String) args[2];
+            return new ThrowingRankBuilder(rankWindowSize, field, throwingType);
+        });
+
+        static {
+            PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD);
+            PARSER.declareString(constructorArg(), FIELD_FIELD);
+            PARSER.declareString(constructorArg(), THROWING_TYPE_FIELD);
+        }
+
+        public static FieldBasedRankBuilder fromXContent(XContentParser parser) throws IOException {
+            return PARSER.parse(parser, null);
+        }
+
+        public ThrowingRankBuilder(final int rankWindowSize, final String field, final String throwingType) {
+            super(rankWindowSize, field);
+            this.throwingRankBuilderType = ThrowingRankBuilderType.valueOf(throwingType);
+        }
+
+        public ThrowingRankBuilder(StreamInput in) throws IOException {
+            super(in);
+            this.throwingRankBuilderType = in.readEnum(ThrowingRankBuilderType.class);
+        }
+
+        @Override
+        protected void doWriteTo(StreamOutput out) throws IOException {
+            super.doWriteTo(out);
+            out.writeEnum(throwingRankBuilderType);
+        }
+
+        @Override
+        protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+            super.doXContent(builder, params);
+            builder.field(THROWING_TYPE_FIELD.getPreferredName(), throwingRankBuilderType);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return "throwing-rank";
+        }
+
+        @Override
+        public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
+            if (this.throwingRankBuilderType == ThrowingRankBuilderType.THROWING_QUERY_PHASE_SHARD_CONTEXT)
+                return new QueryPhaseRankShardContext(queries, rankWindowSize()) {
+                    @Override
+                    public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
+                        throw new UnsupportedOperationException("qps - simulated failure");
+                    }
+                };
+            else {
+                return super.buildQueryPhaseShardContext(queries, from);
+            }
+        }
+
+        @Override
+        public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) {
+            if (this.throwingRankBuilderType == ThrowingRankBuilderType.THROWING_QUERY_PHASE_COORDINATOR_CONTEXT)
+                return new QueryPhaseRankCoordinatorContext(rankWindowSize()) {
+                    @Override
+                    public ScoreDoc[] rankQueryPhaseResults(
+                        List<QuerySearchResult> querySearchResults,
+                        SearchPhaseController.TopDocsStats topDocStats
+                    ) {
+                        throw new UnsupportedOperationException("qpc - simulated failure");
+                    }
+                };
+            else {
+                return super.buildQueryPhaseCoordinatorContext(size, from);
+            }
+        }
+
+        @Override
+        public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+            if (this.throwingRankBuilderType == ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_SHARD_CONTEXT)
+                return new RankFeaturePhaseRankShardContext(field) {
+                    @Override
+                    public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) {
+                        throw new UnsupportedOperationException("rfs - simulated failure");
+                    }
+                };
+            else {
+                return super.buildRankFeaturePhaseShardContext();
+            }
+        }
+
+        @Override
+        public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from) {
+            if (this.throwingRankBuilderType == ThrowingRankBuilderType.THROWING_RANK_FEATURE_PHASE_COORDINATOR_CONTEXT)
+                return new RankFeaturePhaseRankCoordinatorContext(size, from, rankWindowSize()) {
+                    @Override
+                    protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
+                        throw new UnsupportedOperationException("rfc - simulated failure");
+                    }
+                };
+            else {
+                return super.buildRankFeaturePhaseCoordinatorContext(size, from);
+            }
+        }
+    }
+
+    public static class FieldBasedRerankerPlugin extends Plugin implements SearchPlugin {
+
+        private static final String FIELD_BASED_RANK_BUILDER_NAME = "field-based-rank";
+        private static final String THROWING_RANK_BUILDER_NAME = "throwing-rank";
+
+        @Override
+        public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+            return List.of(
+                new NamedWriteableRegistry.Entry(RankBuilder.class, FIELD_BASED_RANK_BUILDER_NAME, FieldBasedRankBuilder::new),
+                new NamedWriteableRegistry.Entry(RankBuilder.class, THROWING_RANK_BUILDER_NAME, ThrowingRankBuilder::new),
+                new NamedWriteableRegistry.Entry(RankShardResult.class, "rank_feature_shard", RankFeatureShardResult::new)
+            );
+        }
+
+        @Override
+        public List<NamedXContentRegistry.Entry> getNamedXContent() {
+            return List.of(
+                new NamedXContentRegistry.Entry(
+                    RankBuilder.class,
+                    new ParseField(FIELD_BASED_RANK_BUILDER_NAME),
+                    FieldBasedRankBuilder::fromXContent
+                ),
+                new NamedXContentRegistry.Entry(
+                    RankBuilder.class,
+                    new ParseField(THROWING_RANK_BUILDER_NAME),
+                    ThrowingRankBuilder::fromXContent
+                )
+            );
+        }
+    }
+}
diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java
index d8682500c49d6..2f08129b4080d 100644
--- a/server/src/main/java/module-info.java
+++ b/server/src/main/java/module-info.java
@@ -362,6 +362,7 @@
     exports org.elasticsearch.search.query;
     exports org.elasticsearch.search.rank;
     exports org.elasticsearch.search.rank.context;
+    exports org.elasticsearch.search.rank.feature;
     exports org.elasticsearch.search.rescore;
     exports org.elasticsearch.search.retriever;
     exports org.elasticsearch.search.runtime;
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index e8a33217b937d..72771855ff622 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -184,7 +184,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_EMBEDDINGS_ADDED = def(8_675_00_0);
     public static final TransportVersion ADD_MISTRAL_EMBEDDINGS_INFERENCE = def(8_676_00_0);
     public static final TransportVersion ML_CHUNK_INFERENCE_OPTION = def(8_677_00_0);
-
+    public static final TransportVersion RANK_FEATURE_PHASE_ADDED = def(8_678_00_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _
diff --git a/server/src/main/java/org/elasticsearch/action/search/CCSSingleCoordinatorSearchProgressListener.java b/server/src/main/java/org/elasticsearch/action/search/CCSSingleCoordinatorSearchProgressListener.java
index 3b594c94db9a7..0504d0cde8986 100644
--- a/server/src/main/java/org/elasticsearch/action/search/CCSSingleCoordinatorSearchProgressListener.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CCSSingleCoordinatorSearchProgressListener.java
@@ -260,6 +260,24 @@ public void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, Interna
         }
     }
 
+    /**
+     * Executed when a shard returns a rank feature result.
+     *
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}.
+     */
+    @Override
+    public void onRankFeatureResult(int shardIndex) {}
+
+    /**
+     * Executed when a shard reports a rank feature failure.
+     *
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}.
+     * @param shardTarget The last shard target that thrown an exception.
+     * @param exc The cause of the failure.
+     */
+    @Override
+    public void onRankFeatureFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {}
+
     /**
      * Executed when a shard returns a fetch result.
      *
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index f804ab31faf8e..2308f5fcc8085 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -17,8 +17,6 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.ShardSearchContextId;
-import org.elasticsearch.search.query.QuerySearchResult;
-import org.elasticsearch.transport.Transport;
 
 import java.util.List;
 import java.util.function.BiFunction;
@@ -29,7 +27,7 @@
  */
 final class FetchSearchPhase extends SearchPhase {
     private final ArraySearchPhaseResults<FetchSearchResult> fetchResults;
-    private final AtomicArray<SearchPhaseResult> queryResults;
+    private final AtomicArray<SearchPhaseResult> searchPhaseShardResults;
     private final BiFunction<SearchResponseSections, AtomicArray<SearchPhaseResult>, SearchPhase> nextPhaseFactory;
     private final SearchPhaseContext context;
     private final Logger logger;
@@ -74,7 +72,7 @@ final class FetchSearchPhase extends SearchPhase {
         }
         this.fetchResults = new ArraySearchPhaseResults<>(resultConsumer.getNumShards());
         context.addReleasable(fetchResults);
-        this.queryResults = resultConsumer.getAtomicArray();
+        this.searchPhaseShardResults = resultConsumer.getAtomicArray();
         this.aggregatedDfs = aggregatedDfs;
         this.nextPhaseFactory = nextPhaseFactory;
         this.context = context;
@@ -103,19 +101,20 @@ private void innerRun() {
         final int numShards = context.getNumShards();
         // Usually when there is a single shard, we force the search type QUERY_THEN_FETCH. But when there's kNN, we might
         // still use DFS_QUERY_THEN_FETCH, which does not perform the "query and fetch" optimization during the query phase.
-        final boolean queryAndFetchOptimization = queryResults.length() == 1
+        final boolean queryAndFetchOptimization = searchPhaseShardResults.length() == 1
             && context.getRequest().hasKnnSearch() == false
-            && reducedQueryPhase.rankCoordinatorContext() == null;
+            && reducedQueryPhase.queryPhaseRankCoordinatorContext() == null;
         if (queryAndFetchOptimization) {
             assert assertConsistentWithQueryAndFetchOptimization();
             // query AND fetch optimization
-            moveToNextPhase(queryResults);
+            moveToNextPhase(searchPhaseShardResults);
         } else {
             ScoreDoc[] scoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs();
             // no docs to fetch -- sidestep everything and return
             if (scoreDocs.length == 0) {
                 // we have to release contexts here to free up resources
-                queryResults.asList().stream().map(SearchPhaseResult::queryResult).forEach(this::releaseIrrelevantSearchContext);
+                searchPhaseShardResults.asList()
+                    .forEach(searchPhaseShardResult -> releaseIrrelevantSearchContext(searchPhaseShardResult, context));
                 moveToNextPhase(fetchResults.getAtomicArray());
             } else {
                 final ScoreDoc[] lastEmittedDocPerShard = context.getRequest().scroll() != null
@@ -130,19 +129,19 @@ private void innerRun() {
                 );
                 for (int i = 0; i < docIdsToLoad.length; i++) {
                     List<Integer> entry = docIdsToLoad[i];
-                    SearchPhaseResult queryResult = queryResults.get(i);
+                    SearchPhaseResult shardPhaseResult = searchPhaseShardResults.get(i);
                     if (entry == null) { // no results for this shard ID
-                        if (queryResult != null) {
+                        if (shardPhaseResult != null) {
                             // if we got some hits from this shard we have to release the context there
                             // we do this as we go since it will free up resources and passing on the request on the
                             // transport layer is cheap.
-                            releaseIrrelevantSearchContext(queryResult.queryResult());
+                            releaseIrrelevantSearchContext(shardPhaseResult, context);
                             progressListener.notifyFetchResult(i);
                         }
                         // in any case we count down this result since we don't talk to this shard anymore
                         counter.countDown();
                     } else {
-                        executeFetch(queryResult, counter, entry, (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[i] : null);
+                        executeFetch(shardPhaseResult, counter, entry, (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[i] : null);
                     }
                 }
             }
@@ -150,31 +149,33 @@ private void innerRun() {
     }
 
     private boolean assertConsistentWithQueryAndFetchOptimization() {
-        var phaseResults = queryResults.asList();
+        var phaseResults = searchPhaseShardResults.asList();
         assert phaseResults.isEmpty() || phaseResults.get(0).fetchResult() != null
             : "phaseResults empty [" + phaseResults.isEmpty() + "], single result: " + phaseResults.get(0).fetchResult();
         return true;
     }
 
     private void executeFetch(
-        SearchPhaseResult queryResult,
+        SearchPhaseResult shardPhaseResult,
         final CountedCollector<FetchSearchResult> counter,
         final List<Integer> entry,
         ScoreDoc lastEmittedDocForShard
     ) {
-        final SearchShardTarget shardTarget = queryResult.getSearchShardTarget();
-        final int shardIndex = queryResult.getShardIndex();
-        final ShardSearchContextId contextId = queryResult.queryResult().getContextId();
+        final SearchShardTarget shardTarget = shardPhaseResult.getSearchShardTarget();
+        final int shardIndex = shardPhaseResult.getShardIndex();
+        final ShardSearchContextId contextId = shardPhaseResult.queryResult() != null
+            ? shardPhaseResult.queryResult().getContextId()
+            : shardPhaseResult.rankFeatureResult().getContextId();
         context.getSearchTransport()
             .sendExecuteFetch(
                 context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()),
                 new ShardFetchSearchRequest(
-                    context.getOriginalIndices(queryResult.getShardIndex()),
+                    context.getOriginalIndices(shardPhaseResult.getShardIndex()),
                     contextId,
-                    queryResult.getShardSearchRequest(),
+                    shardPhaseResult.getShardSearchRequest(),
                     entry,
                     lastEmittedDocForShard,
-                    queryResult.getRescoreDocIds(),
+                    shardPhaseResult.getRescoreDocIds(),
                     aggregatedDfs
                 ),
                 context.getTask(),
@@ -199,40 +200,17 @@ public void onFailure(Exception e) {
                             // the search context might not be cleared on the node where the fetch was executed for example
                             // because the action was rejected by the thread pool. in this case we need to send a dedicated
                             // request to clear the search context.
-                            releaseIrrelevantSearchContext(queryResult.queryResult());
+                            releaseIrrelevantSearchContext(shardPhaseResult, context);
                         }
                     }
                 }
             );
     }
 
-    /**
-     * Releases shard targets that are not used in the docsIdsToLoad.
-     */
-    private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) {
-        // we only release search context that we did not fetch from, if we are not scrolling
-        // or using a PIT and if it has at least one hit that didn't make it to the global topDocs
-        if (queryResult.hasSearchContext()
-            && context.getRequest().scroll() == null
-            && (context.isPartOfPointInTime(queryResult.getContextId()) == false)) {
-            try {
-                SearchShardTarget shardTarget = queryResult.getSearchShardTarget();
-                Transport.Connection connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId());
-                context.sendReleaseSearchContext(
-                    queryResult.getContextId(),
-                    connection,
-                    context.getOriginalIndices(queryResult.getShardIndex())
-                );
-            } catch (Exception e) {
-                logger.trace("failed to release context", e);
-            }
-        }
-    }
-
     private void moveToNextPhase(AtomicArray<? extends SearchPhaseResult> fetchResultsArr) {
         var resp = SearchPhaseController.merge(context.getRequest().scroll() != null, reducedQueryPhase, fetchResultsArr);
         context.addReleasable(resp::decRef);
         fetchResults.close();
-        context.executeNextPhase(this, nextPhaseFactory.apply(resp, queryResults));
+        context.executeNextPhase(this, nextPhaseFactory.apply(resp, searchPhaseShardResults));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
index 767597625edc6..291982dd9bdd3 100644
--- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java
@@ -7,23 +7,39 @@
  */
 package org.elasticsearch.action.search;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.search.ScoreDoc;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.dfs.AggregatedDfs;
+import org.elasticsearch.search.internal.ShardSearchContextId;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.feature.RankFeatureDoc;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
+import org.elasticsearch.search.rank.feature.RankFeatureShardRequest;
+
+import java.util.List;
 
 /**
  * This search phase is responsible for executing any re-ranking needed for the given search request, iff that is applicable.
- * It starts by retrieving {code num_shards * window_size} results from the query phase and reduces them to a global list of
+ * It starts by retrieving {@code num_shards * window_size} results from the query phase and reduces them to a global list of
  * the top {@code window_size} results. It then reaches out to the shards to extract the needed feature data,
  * and finally passes all this information to the appropriate {@code RankFeatureRankCoordinatorContext} which is responsible for reranking
  * the results. If no rank query is specified, it proceeds directly to the next phase (FetchSearchPhase) by first reducing the results.
  */
-public final class RankFeaturePhase extends SearchPhase {
+public class RankFeaturePhase extends SearchPhase {
 
+    private static final Logger logger = LogManager.getLogger(RankFeaturePhase.class);
     private final SearchPhaseContext context;
-    private final SearchPhaseResults<SearchPhaseResult> queryPhaseResults;
-
+    final SearchPhaseResults<SearchPhaseResult> queryPhaseResults;
+    final SearchPhaseResults<SearchPhaseResult> rankPhaseResults;
     private final AggregatedDfs aggregatedDfs;
+    private final SearchProgressListener progressListener;
 
     RankFeaturePhase(SearchPhaseResults<SearchPhaseResult> queryPhaseResults, AggregatedDfs aggregatedDfs, SearchPhaseContext context) {
         super("rank-feature");
@@ -38,6 +54,9 @@ public final class RankFeaturePhase extends SearchPhase {
         this.context = context;
         this.queryPhaseResults = queryPhaseResults;
         this.aggregatedDfs = aggregatedDfs;
+        this.rankPhaseResults = new ArraySearchPhaseResults<>(context.getNumShards());
+        context.addReleasable(rankPhaseResults);
+        this.progressListener = context.getTask().getProgressListener();
     }
 
     @Override
@@ -59,16 +78,154 @@ public void onFailure(Exception e) {
         });
     }
 
-    private void innerRun() throws Exception {
-        // other than running reduce, this is currently close to a no-op
+    void innerRun() throws Exception {
+        // if the RankBuilder specifies a QueryPhaseCoordinatorContext, it will be called as part of the reduce call
+        // to operate on the first `window_size * num_shards` results and merge them appropriately.
         SearchPhaseController.ReducedQueryPhase reducedQueryPhase = queryPhaseResults.reduce();
-        moveToNextPhase(queryPhaseResults, reducedQueryPhase);
+        RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext = coordinatorContext(context.getRequest().source());
+        if (rankFeaturePhaseRankCoordinatorContext != null) {
+            ScoreDoc[] queryScoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs(); // rank_window_size
+            final List<Integer>[] docIdsToLoad = SearchPhaseController.fillDocIdsToLoad(context.getNumShards(), queryScoreDocs);
+            final CountedCollector<SearchPhaseResult> rankRequestCounter = new CountedCollector<>(
+                rankPhaseResults,
+                context.getNumShards(),
+                () -> onPhaseDone(rankFeaturePhaseRankCoordinatorContext, reducedQueryPhase),
+                context
+            );
+
+            // we send out a request to each shard in order to fetch the needed feature info
+            for (int i = 0; i < docIdsToLoad.length; i++) {
+                List<Integer> entry = docIdsToLoad[i];
+                SearchPhaseResult queryResult = queryPhaseResults.getAtomicArray().get(i);
+                if (entry == null || entry.isEmpty()) {
+                    if (queryResult != null) {
+                        releaseIrrelevantSearchContext(queryResult, context);
+                        progressListener.notifyRankFeatureResult(i);
+                    }
+                    rankRequestCounter.countDown();
+                } else {
+                    executeRankFeatureShardPhase(queryResult, rankRequestCounter, entry);
+                }
+            }
+        } else {
+            moveToNextPhase(queryPhaseResults, reducedQueryPhase);
+        }
+    }
+
+    private RankFeaturePhaseRankCoordinatorContext coordinatorContext(SearchSourceBuilder source) {
+        return source == null || source.rankBuilder() == null
+            ? null
+            : context.getRequest()
+                .source()
+                .rankBuilder()
+                .buildRankFeaturePhaseCoordinatorContext(context.getRequest().source().size(), context.getRequest().source().from());
     }
 
-    private void moveToNextPhase(
-        SearchPhaseResults<SearchPhaseResult> phaseResults,
+    private void executeRankFeatureShardPhase(
+        SearchPhaseResult queryResult,
+        final CountedCollector<SearchPhaseResult> rankRequestCounter,
+        final List<Integer> entry
+    ) {
+        final SearchShardTarget shardTarget = queryResult.queryResult().getSearchShardTarget();
+        final ShardSearchContextId contextId = queryResult.queryResult().getContextId();
+        final int shardIndex = queryResult.getShardIndex();
+        context.getSearchTransport()
+            .sendExecuteRankFeature(
+                context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()),
+                new RankFeatureShardRequest(
+                    context.getOriginalIndices(queryResult.getShardIndex()),
+                    queryResult.getContextId(),
+                    queryResult.getShardSearchRequest(),
+                    entry
+                ),
+                context.getTask(),
+                new SearchActionListener<>(shardTarget, shardIndex) {
+                    @Override
+                    protected void innerOnResponse(RankFeatureResult response) {
+                        try {
+                            progressListener.notifyRankFeatureResult(shardIndex);
+                            rankRequestCounter.onResult(response);
+                        } catch (Exception e) {
+                            context.onPhaseFailure(RankFeaturePhase.this, "", e);
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        try {
+                            logger.debug(() -> "[" + contextId + "] Failed to execute rank phase", e);
+                            progressListener.notifyRankFeatureFailure(shardIndex, shardTarget, e);
+                            rankRequestCounter.onFailure(shardIndex, shardTarget, e);
+                        } finally {
+                            releaseIrrelevantSearchContext(queryResult, context);
+                        }
+                    }
+                }
+            );
+    }
+
+    private void onPhaseDone(
+        RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext,
         SearchPhaseController.ReducedQueryPhase reducedQueryPhase
     ) {
+        assert rankFeaturePhaseRankCoordinatorContext != null;
+        ThreadedActionListener<RankFeatureDoc[]> rankResultListener = new ThreadedActionListener<>(context, new ActionListener<>() {
+            @Override
+            public void onResponse(RankFeatureDoc[] docsWithUpdatedScores) {
+                RankFeatureDoc[] topResults = rankFeaturePhaseRankCoordinatorContext.rankAndPaginate(docsWithUpdatedScores);
+                SearchPhaseController.ReducedQueryPhase reducedRankFeaturePhase = newReducedQueryPhaseResults(
+                    reducedQueryPhase,
+                    topResults
+                );
+                moveToNextPhase(rankPhaseResults, reducedRankFeaturePhase);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                context.onPhaseFailure(RankFeaturePhase.this, "Computing updated ranks for results failed", e);
+            }
+        });
+        rankFeaturePhaseRankCoordinatorContext.rankGlobalResults(
+            rankPhaseResults.getAtomicArray().asList().stream().map(SearchPhaseResult::rankFeatureResult).toList(),
+            rankResultListener
+        );
+    }
+
+    private SearchPhaseController.ReducedQueryPhase newReducedQueryPhaseResults(
+        SearchPhaseController.ReducedQueryPhase reducedQueryPhase,
+        ScoreDoc[] scoreDocs
+    ) {
+
+        return new SearchPhaseController.ReducedQueryPhase(
+            reducedQueryPhase.totalHits(),
+            reducedQueryPhase.fetchHits(),
+            maxScore(scoreDocs),
+            reducedQueryPhase.timedOut(),
+            reducedQueryPhase.terminatedEarly(),
+            reducedQueryPhase.suggest(),
+            reducedQueryPhase.aggregations(),
+            reducedQueryPhase.profileBuilder(),
+            new SearchPhaseController.SortedTopDocs(scoreDocs, false, null, null, null, 0),
+            reducedQueryPhase.sortValueFormats(),
+            reducedQueryPhase.queryPhaseRankCoordinatorContext(),
+            reducedQueryPhase.numReducePhases(),
+            reducedQueryPhase.size(),
+            reducedQueryPhase.from(),
+            reducedQueryPhase.isEmptyResult()
+        );
+    }
+
+    private float maxScore(ScoreDoc[] scoreDocs) {
+        float maxScore = Float.NaN;
+        for (ScoreDoc scoreDoc : scoreDocs) {
+            if (Float.isNaN(maxScore) || scoreDoc.score > maxScore) {
+                maxScore = scoreDoc.score;
+            }
+        }
+        return maxScore;
+    }
+
+    void moveToNextPhase(SearchPhaseResults<SearchPhaseResult> phaseResults, SearchPhaseController.ReducedQueryPhase reducedQueryPhase) {
         context.executeNextPhase(this, new FetchSearchPhase(phaseResults, aggregatedDfs, context, reducedQueryPhase));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
index 9d3eadcc42bf9..5ed449667fe57 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java
@@ -9,6 +9,9 @@
 
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.transport.Transport;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -62,4 +65,35 @@ static void doCheckNoMissingShards(String phaseName, SearchRequest request, Grou
             }
         }
     }
+
+    /**
+     * Releases shard targets that are not used in the docsIdsToLoad.
+     */
+    protected void releaseIrrelevantSearchContext(SearchPhaseResult searchPhaseResult, SearchPhaseContext context) {
+        // we only release search context that we did not fetch from, if we are not scrolling
+        // or using a PIT and if it has at least one hit that didn't make it to the global topDocs
+        if (searchPhaseResult == null) {
+            return;
+        }
+        // phaseResult.getContextId() is the same for query & rank feature results
+        SearchPhaseResult phaseResult = searchPhaseResult.queryResult() != null
+            ? searchPhaseResult.queryResult()
+            : searchPhaseResult.rankFeatureResult();
+        if (phaseResult != null
+            && phaseResult.hasSearchContext()
+            && context.getRequest().scroll() == null
+            && (context.isPartOfPointInTime(phaseResult.getContextId()) == false)) {
+            try {
+                SearchShardTarget shardTarget = phaseResult.getSearchShardTarget();
+                Transport.Connection connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId());
+                context.sendReleaseSearchContext(
+                    phaseResult.getContextId(),
+                    connection,
+                    context.getOriginalIndices(phaseResult.getShardIndex())
+                );
+            } catch (Exception e) {
+                context.getLogger().trace("failed to release context", e);
+            }
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index 1b894dfe3d8bd..1d3859b9038fe 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -456,7 +456,7 @@ private static SearchHits getHits(
                     : "not enough hits fetched. index [" + index + "] length: " + fetchResult.hits().getHits().length;
                 SearchHit searchHit = fetchResult.hits().getHits()[index];
                 searchHit.shard(fetchResult.getSearchShardTarget());
-                if (reducedQueryPhase.rankCoordinatorContext != null) {
+                if (reducedQueryPhase.queryPhaseRankCoordinatorContext != null) {
                     assert shardDoc instanceof RankDoc;
                     searchHit.setRank(((RankDoc) shardDoc).rank);
                     searchHit.score(shardDoc.score);
@@ -747,7 +747,7 @@ public record ReducedQueryPhase(
         // sort value formats used to sort / format the result
         DocValueFormat[] sortValueFormats,
         // the rank context if ranking is used
-        QueryPhaseRankCoordinatorContext rankCoordinatorContext,
+        QueryPhaseRankCoordinatorContext queryPhaseRankCoordinatorContext,
         // the number of reduces phases
         int numReducePhases,
         // the size of the top hits to return
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java
index f5d280a01257c..3b5e03cb5ac4a 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java
@@ -88,6 +88,22 @@ protected void onPartialReduce(List<SearchShard> shards, TotalHits totalHits, In
      */
     protected void onFinalReduce(List<SearchShard> shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {}
 
+    /**
+     * Executed when a shard returns a rank feature result.
+     *
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}.
+     */
+    protected void onRankFeatureResult(int shardIndex) {}
+
+    /**
+     * Executed when a shard reports a rank feature failure.
+     *
+     * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}.
+     * @param shardTarget The last shard target that thrown an exception.
+     * @param exc The cause of the failure.
+     */
+    protected void onRankFeatureFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {}
+
     /**
      * Executed when a shard returns a fetch result.
      *
@@ -160,6 +176,22 @@ protected final void notifyFinalReduce(List<SearchShard> shards, TotalHits total
         }
     }
 
+    final void notifyRankFeatureResult(int shardIndex) {
+        try {
+            onRankFeatureResult(shardIndex);
+        } catch (Exception e) {
+            logger.warn(() -> "[" + shards.get(shardIndex) + "] Failed to execute progress listener on rank-feature result", e);
+        }
+    }
+
+    final void notifyRankFeatureFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
+        try {
+            onRankFeatureFailure(shardIndex, shardTarget, exc);
+        } catch (Exception e) {
+            logger.warn(() -> "[" + shards.get(shardIndex) + "] Failed to execute progress listener on rank-feature failure", e);
+        }
+    }
+
     final void notifyFetchResult(int shardIndex) {
         try {
             onFetchResult(shardIndex);
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 4e3fdbc9633b9..3e4f6dfec9fdb 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -407,7 +407,7 @@ public ActionRequestValidationException validate() {
                     );
                 }
                 int queryCount = source.subSearches().size() + source.knnSearch().size();
-                if (queryCount < 2) {
+                if (source.rankBuilder().isCompoundBuilder() && queryCount < 2) {
                     validationException = addValidationError(
                         "[rank] requires a minimum of [2] result sets using a combination of sub searches and/or knn searches",
                         validationException
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportAPMMetrics.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportAPMMetrics.java
index 93b8e22d0d7cd..9f8896f169350 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportAPMMetrics.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportAPMMetrics.java
@@ -19,6 +19,7 @@ public class SearchTransportAPMMetrics {
     public static final String DFS_ACTION_METRIC = "dfs_query_then_fetch/shard_dfs_phase";
     public static final String QUERY_ID_ACTION_METRIC = "dfs_query_then_fetch/shard_query_phase";
     public static final String QUERY_ACTION_METRIC = "query_then_fetch/shard_query_phase";
+    public static final String RANK_SHARD_FEATURE_ACTION_METRIC = "rank/shard_feature_phase";
     public static final String FREE_CONTEXT_ACTION_METRIC = "shard_release_context";
     public static final String FETCH_ID_ACTION_METRIC = "shard_fetch_phase";
     public static final String QUERY_SCROLL_ACTION_METRIC = "scroll/shard_query_phase";
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 66c395cf51d96..d627da9b0e33b 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -39,6 +39,8 @@
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.query.ScrollQuerySearchResult;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
+import org.elasticsearch.search.rank.feature.RankFeatureShardRequest;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteClusterService;
@@ -70,6 +72,7 @@
 import static org.elasticsearch.action.search.SearchTransportAPMMetrics.QUERY_FETCH_SCROLL_ACTION_METRIC;
 import static org.elasticsearch.action.search.SearchTransportAPMMetrics.QUERY_ID_ACTION_METRIC;
 import static org.elasticsearch.action.search.SearchTransportAPMMetrics.QUERY_SCROLL_ACTION_METRIC;
+import static org.elasticsearch.action.search.SearchTransportAPMMetrics.RANK_SHARD_FEATURE_ACTION_METRIC;
 
 /**
  * An encapsulation of {@link org.elasticsearch.search.SearchService} operations exposed through
@@ -96,6 +99,8 @@ public class SearchTransportService {
     public static final String FETCH_ID_SCROLL_ACTION_NAME = "indices:data/read/search[phase/fetch/id/scroll]";
     public static final String FETCH_ID_ACTION_NAME = "indices:data/read/search[phase/fetch/id]";
 
+    public static final String RANK_FEATURE_SHARD_ACTION_NAME = "indices:data/read/search[phase/rank/feature]";
+
     /**
      * The Can-Match phase. It is executed to pre-filter shards that a search request hits. It rewrites the query on
      * the shard and checks whether the result of the rewrite matches no documents, in which case the shard can be
@@ -250,6 +255,21 @@ public void sendExecuteScrollQuery(
         );
     }
 
+    public void sendExecuteRankFeature(
+        Transport.Connection connection,
+        final RankFeatureShardRequest request,
+        SearchTask task,
+        final SearchActionListener<RankFeatureResult> listener
+    ) {
+        transportService.sendChildRequest(
+            connection,
+            RANK_FEATURE_SHARD_ACTION_NAME,
+            request,
+            task,
+            new ConnectionCountingHandler<>(listener, RankFeatureResult::new, connection)
+        );
+    }
+
     public void sendExecuteScrollFetch(
         Transport.Connection connection,
         final InternalScrollSearchRequest request,
@@ -539,6 +559,16 @@ public static void registerRequestHandler(
         );
         TransportActionProxy.registerProxyAction(transportService, QUERY_FETCH_SCROLL_ACTION_NAME, true, ScrollQueryFetchSearchResult::new);
 
+        final TransportRequestHandler<RankFeatureShardRequest> rankShardFeatureRequest = (request, channel, task) -> searchService
+            .executeRankFeaturePhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel));
+        transportService.registerRequestHandler(
+            RANK_FEATURE_SHARD_ACTION_NAME,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE,
+            RankFeatureShardRequest::new,
+            instrumentedHandler(RANK_SHARD_FEATURE_ACTION_METRIC, transportService, searchTransportMetrics, rankShardFeatureRequest)
+        );
+        TransportActionProxy.registerProxyAction(transportService, RANK_FEATURE_SHARD_ACTION_NAME, true, RankFeatureResult::new);
+
         final TransportRequestHandler<ShardFetchRequest> shardFetchRequestHandler = (request, channel, task) -> searchService
             .executeFetchPhase(request, (SearchShardTask) task, new ChannelActionListener<>(channel));
         transportService.registerRequestHandler(
diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
index e0ca0f7a48cdd..fd2aabce8e952 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
@@ -1044,6 +1044,7 @@ record PluginServiceInstances(
             threadPool,
             scriptService,
             bigArrays,
+            searchModule.getRankFeatureShardPhase(),
             searchModule.getFetchPhase(),
             responseCollectorService,
             circuitBreakerService,
diff --git a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java
index ab90ca42bca98..914dd51d0c6b2 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.fetch.FetchPhase;
+import org.elasticsearch.search.rank.feature.RankFeatureShardPhase;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -116,6 +117,7 @@ SearchService newSearchService(
         ThreadPool threadPool,
         ScriptService scriptService,
         BigArrays bigArrays,
+        RankFeatureShardPhase rankFeatureShardPhase,
         FetchPhase fetchPhase,
         ResponseCollectorService responseCollectorService,
         CircuitBreakerService circuitBreakerService,
@@ -128,6 +130,7 @@ SearchService newSearchService(
             threadPool,
             scriptService,
             bigArrays,
+            rankFeatureShardPhase,
             fetchPhase,
             responseCollectorService,
             circuitBreakerService,
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 9bacf19a9169d..4f16d3a5720fb 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -70,6 +70,7 @@
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.sort.SortAndFormats;
@@ -102,6 +103,7 @@ final class DefaultSearchContext extends SearchContext {
     private final ContextIndexSearcher searcher;
     private DfsSearchResult dfsResult;
     private QuerySearchResult queryResult;
+    private RankFeatureResult rankFeatureResult;
     private FetchSearchResult fetchResult;
     private final float queryBoost;
     private final boolean lowLevelCancellation;
@@ -308,6 +310,17 @@ static boolean isParallelCollectionSupportedForResults(
         return false;
     }
 
+    @Override
+    public void addRankFeatureResult() {
+        this.rankFeatureResult = new RankFeatureResult(this.readerContext.id(), this.shardTarget, this.request);
+        addReleasable(rankFeatureResult::decRef);
+    }
+
+    @Override
+    public RankFeatureResult rankFeatureResult() {
+        return rankFeatureResult;
+    }
+
     @Override
     public void addFetchResult() {
         this.fetchResult = new FetchSearchResult(this.readerContext.id(), this.shardTarget);
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index 8d5fa0a7ac155..d93ff91a6ffe4 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -226,6 +226,7 @@
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.PlainHighlighter;
 import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.search.rank.feature.RankFeatureShardPhase;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.retriever.KnnRetrieverBuilder;
@@ -1252,6 +1253,10 @@ private void registerQuery(QuerySpec<?> spec) {
         );
     }
 
+    public RankFeatureShardPhase getRankFeatureShardPhase() {
+        return new RankFeatureShardPhase();
+    }
+
     public FetchPhase getFetchPhase() {
         return new FetchPhase(fetchSubPhases);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java b/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
index 254cd7d3370b5..450b98b22f39c 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchPhaseResult.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
 import org.elasticsearch.transport.TransportResponse;
 
 import java.io.IOException;
@@ -43,6 +44,14 @@ protected SearchPhaseResult(StreamInput in) throws IOException {
         super(in);
     }
 
+    /**
+     * Specifies whether the specific search phase results are associated with an opened SearchContext on the shards that
+     * executed the request.
+     */
+    public boolean hasSearchContext() {
+        return false;
+    }
+
     /**
      * Returns the search context ID that is used to reference the search context on the executing node
      * or <code>null</code> if no context was created.
@@ -81,6 +90,13 @@ public QuerySearchResult queryResult() {
         return null;
     }
 
+    /**
+     * Returns the rank feature result iff it's included in this response otherwise <code>null</code>
+     */
+    public RankFeatureResult rankFeatureResult() {
+        return null;
+    }
+
     /**
      * Returns the fetch result iff it's included in this response otherwise <code>null</code>
      */
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 41796967c3870..3f9dd7895f6a7 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -112,6 +112,9 @@
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.query.ScrollQuerySearchResult;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
+import org.elasticsearch.search.rank.feature.RankFeatureShardPhase;
+import org.elasticsearch.search.rank.feature.RankFeatureShardRequest;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -151,6 +154,7 @@
 import static org.elasticsearch.core.TimeValue.timeValueMillis;
 import static org.elasticsearch.core.TimeValue.timeValueMinutes;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
+import static org.elasticsearch.search.rank.feature.RankFeatureShardPhase.EMPTY_RESULT;
 
 public class SearchService extends AbstractLifecycleComponent implements IndexEventListener {
     private static final Logger logger = LogManager.getLogger(SearchService.class);
@@ -276,6 +280,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
     private final DfsPhase dfsPhase = new DfsPhase();
 
     private final FetchPhase fetchPhase;
+    private final RankFeatureShardPhase rankFeatureShardPhase;
     private volatile boolean enableSearchWorkerThreads;
     private volatile boolean enableQueryPhaseParallelCollection;
 
@@ -314,6 +319,7 @@ public SearchService(
         ThreadPool threadPool,
         ScriptService scriptService,
         BigArrays bigArrays,
+        RankFeatureShardPhase rankFeatureShardPhase,
         FetchPhase fetchPhase,
         ResponseCollectorService responseCollectorService,
         CircuitBreakerService circuitBreakerService,
@@ -327,6 +333,7 @@ public SearchService(
         this.scriptService = scriptService;
         this.responseCollectorService = responseCollectorService;
         this.bigArrays = bigArrays;
+        this.rankFeatureShardPhase = rankFeatureShardPhase;
         this.fetchPhase = fetchPhase;
         this.multiBucketConsumerService = new MultiBucketConsumerService(
             clusterService,
@@ -713,6 +720,32 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchSh
         }
     }
 
+    public void executeRankFeaturePhase(RankFeatureShardRequest request, SearchShardTask task, ActionListener<RankFeatureResult> listener) {
+        final ReaderContext readerContext = findReaderContext(request.contextId(), request);
+        final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.getShardSearchRequest());
+        final Releasable markAsUsed = readerContext.markAsUsed(getKeepAlive(shardSearchRequest));
+        runAsync(getExecutor(readerContext.indexShard()), () -> {
+            try (SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, ResultsType.RANK_FEATURE, false)) {
+                int[] docIds = request.getDocIds();
+                if (docIds == null || docIds.length == 0) {
+                    searchContext.rankFeatureResult().shardResult(EMPTY_RESULT);
+                    searchContext.rankFeatureResult().incRef();
+                    return searchContext.rankFeatureResult();
+                }
+                rankFeatureShardPhase.prepareForFetch(searchContext, request);
+                fetchPhase.execute(searchContext, docIds);
+                rankFeatureShardPhase.processFetch(searchContext);
+                var rankFeatureResult = searchContext.rankFeatureResult();
+                rankFeatureResult.incRef();
+                return rankFeatureResult;
+            } catch (Exception e) {
+                assert TransportActions.isShardNotAvailableException(e) == false : new AssertionError(e);
+                // we handle the failure in the failure listener below
+                throw e;
+            }
+        }, wrapFailureListener(listener, readerContext, markAsUsed));
+    }
+
     private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchContext context, long afterQueryTime) {
         try (
             Releasable scope = tracer.withScope(context.getTask());
@@ -1559,6 +1592,12 @@ void addResultsObject(SearchContext context) {
                 context.addQueryResult();
             }
         },
+        RANK_FEATURE {
+            @Override
+            void addResultsObject(SearchContext context) {
+                context.addRankFeatureResult();
+            }
+        },
         FETCH {
             @Override
             void addResultsObject(SearchContext context) {
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
index 0c54e8ff89589..4ba191794413d 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -98,7 +98,6 @@ public Source getSource(LeafReaderContext ctx, int doc) {
     }
 
     private SearchHits buildSearchHits(SearchContext context, int[] docIdsToLoad, Profiler profiler) {
-
         FetchContext fetchContext = new FetchContext(context);
         SourceLoader sourceLoader = context.newSourceLoader();
 
diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index d5c3c00c00ce1..e32397e25d773 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
@@ -374,6 +375,16 @@ public float getMaxScore() {
         return in.getMaxScore();
     }
 
+    @Override
+    public void addRankFeatureResult() {
+        in.addRankFeatureResult();
+    }
+
+    @Override
+    public RankFeatureResult rankFeatureResult() {
+        return in.rankFeatureResult();
+    }
+
     @Override
     public FetchSearchResult fetchResult() {
         return in.fetchResult();
diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 35f96ee2dc102..9bc622034184c 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
@@ -332,6 +333,10 @@ public Query rewrittenQuery() {
 
     public abstract float getMaxScore();
 
+    public abstract void addRankFeatureResult();
+
+    public abstract RankFeatureResult rankFeatureResult();
+
     public abstract FetchPhase fetchPhase();
 
     public abstract FetchSearchResult fetchResult();
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 828c6d2b4f3e8..0d2610aa34282 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -87,35 +87,38 @@ static void executeRank(SearchContext searchContext) throws QueryPhaseExecutionE
         boolean searchTimedOut = querySearchResult.searchTimedOut();
         long serviceTimeEWMA = querySearchResult.serviceTimeEWMA();
         int nodeQueueSize = querySearchResult.nodeQueueSize();
-
-        // run each of the rank queries
-        for (Query rankQuery : queryPhaseRankShardContext.queries()) {
-            // if a search timeout occurs, exit with partial results
-            if (searchTimedOut) {
-                break;
-            }
-            try (
-                RankSearchContext rankSearchContext = new RankSearchContext(
-                    searchContext,
-                    rankQuery,
-                    queryPhaseRankShardContext.rankWindowSize()
-                )
-            ) {
-                QueryPhase.addCollectorsAndSearch(rankSearchContext);
-                QuerySearchResult rrfQuerySearchResult = rankSearchContext.queryResult();
-                rrfRankResults.add(rrfQuerySearchResult.topDocs().topDocs);
-                serviceTimeEWMA += rrfQuerySearchResult.serviceTimeEWMA();
-                nodeQueueSize = Math.max(nodeQueueSize, rrfQuerySearchResult.nodeQueueSize());
-                searchTimedOut = rrfQuerySearchResult.searchTimedOut();
+        try {
+            // run each of the rank queries
+            for (Query rankQuery : queryPhaseRankShardContext.queries()) {
+                // if a search timeout occurs, exit with partial results
+                if (searchTimedOut) {
+                    break;
+                }
+                try (
+                    RankSearchContext rankSearchContext = new RankSearchContext(
+                        searchContext,
+                        rankQuery,
+                        queryPhaseRankShardContext.rankWindowSize()
+                    )
+                ) {
+                    QueryPhase.addCollectorsAndSearch(rankSearchContext);
+                    QuerySearchResult rrfQuerySearchResult = rankSearchContext.queryResult();
+                    rrfRankResults.add(rrfQuerySearchResult.topDocs().topDocs);
+                    serviceTimeEWMA += rrfQuerySearchResult.serviceTimeEWMA();
+                    nodeQueueSize = Math.max(nodeQueueSize, rrfQuerySearchResult.nodeQueueSize());
+                    searchTimedOut = rrfQuerySearchResult.searchTimedOut();
+                }
             }
-        }
 
-        querySearchResult.setRankShardResult(queryPhaseRankShardContext.combineQueryPhaseResults(rrfRankResults));
+            querySearchResult.setRankShardResult(queryPhaseRankShardContext.combineQueryPhaseResults(rrfRankResults));
 
-        // record values relevant to all queries
-        querySearchResult.searchTimedOut(searchTimedOut);
-        querySearchResult.serviceTimeEWMA(serviceTimeEWMA);
-        querySearchResult.nodeQueueSize(nodeQueueSize);
+            // record values relevant to all queries
+            querySearchResult.searchTimedOut(searchTimedOut);
+            querySearchResult.serviceTimeEWMA(serviceTimeEWMA);
+            querySearchResult.nodeQueueSize(nodeQueueSize);
+        } catch (Exception e) {
+            throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Failed to execute rank query", e);
+        }
     }
 
     static void executeQuery(SearchContext searchContext) throws QueryPhaseExecutionException {
diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java b/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java
index 7118c9f49b36d..f496758c3f5c6 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java
@@ -16,6 +16,8 @@
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -32,7 +34,7 @@ public abstract class RankBuilder implements VersionedNamedWriteable, ToXContent
 
     public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
 
-    public static final int DEFAULT_WINDOW_SIZE = SearchService.DEFAULT_SIZE;
+    public static final int DEFAULT_RANK_WINDOW_SIZE = SearchService.DEFAULT_SIZE;
 
     private final int rankWindowSize;
 
@@ -68,6 +70,12 @@ public int rankWindowSize() {
         return rankWindowSize;
     }
 
+    /**
+     * Specify whether this rank builder is a compound builder or not. A compound builder is a rank builder that requires
+     * two or more queries to be executed in order to generate the final result.
+     */
+    public abstract boolean isCompoundBuilder();
+
     /**
      * Generates a context used to execute required searches during the query phase on the shard.
      */
@@ -78,6 +86,19 @@ public int rankWindowSize() {
      */
     public abstract QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from);
 
+    /**
+     * Generates a context used to execute the rank feature phase on the shard. This is responsible for retrieving any needed
+     * feature data, and passing them back to the coordinator through the appropriate {@link  RankShardResult}.
+     */
+    public abstract RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext();
+
+    /**
+     * Generates a context used to perform global ranking during the RankFeature phase,
+     * on the coordinator based on all the individual shard results. The output of this will be a `size` ranked list of ordered results,
+     * which will then be passed to fetch phase.
+     */
+    public abstract RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from);
+
     @Override
     public final boolean equals(Object obj) {
         if (this == obj) {
diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java b/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java
index 1cb5843dfc7da..7f8e99971d61b 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java
@@ -43,6 +43,7 @@
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
@@ -57,14 +58,14 @@ public class RankSearchContext extends SearchContext {
 
     private final SearchContext parent;
     private final Query rankQuery;
-    private final int windowSize;
+    private final int rankWindowSize;
     private final QuerySearchResult querySearchResult;
 
     @SuppressWarnings("this-escape")
-    public RankSearchContext(SearchContext parent, Query rankQuery, int windowSize) {
+    public RankSearchContext(SearchContext parent, Query rankQuery, int rankWindowSize) {
         this.parent = parent;
         this.rankQuery = parent.buildFilteredQuery(rankQuery);
-        this.windowSize = windowSize;
+        this.rankWindowSize = rankWindowSize;
         this.querySearchResult = new QuerySearchResult(parent.readerContext().id(), parent.shardTarget(), parent.request());
         this.addReleasable(querySearchResult::decRef);
     }
@@ -182,7 +183,7 @@ public int from() {
 
     @Override
     public int size() {
-        return windowSize;
+        return rankWindowSize;
     }
 
     /**
@@ -492,6 +493,16 @@ public FetchPhase fetchPhase() {
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    public void addRankFeatureResult() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public RankFeatureResult rankFeatureResult() {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public FetchSearchResult fetchResult() {
         throw new UnsupportedOperationException();
diff --git a/server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankCoordinatorContext.java b/server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankCoordinatorContext.java
new file mode 100644
index 0000000000000..b8951a4779166
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankCoordinatorContext.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank.context;
+
+import org.apache.lucene.search.ScoreDoc;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.search.rank.feature.RankFeatureDoc;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
+import org.elasticsearch.search.rank.feature.RankFeatureShardResult;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import static org.elasticsearch.search.SearchService.DEFAULT_FROM;
+import static org.elasticsearch.search.SearchService.DEFAULT_SIZE;
+
+/**
+ * {@code RankFeaturePhaseRankCoordinatorContext} is a base class that runs on the coordinating node and is responsible for retrieving
+ * {@code window_size} total results from all shards, rank them, and then produce a final paginated response of [from, from+size] results.
+ */
+public abstract class RankFeaturePhaseRankCoordinatorContext {
+
+    protected final int size;
+    protected final int from;
+    protected final int rankWindowSize;
+
+    public RankFeaturePhaseRankCoordinatorContext(int size, int from, int rankWindowSize) {
+        this.size = size < 0 ? DEFAULT_SIZE : size;
+        this.from = from < 0 ? DEFAULT_FROM : from;
+        this.rankWindowSize = rankWindowSize;
+    }
+
+    /**
+     * Computes the updated scores for a list of features (i.e. document-based data). We also pass along an ActionListener
+     * that should be called with the new scores, and will continue execution to the next phase
+     */
+    protected abstract void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener);
+
+    /**
+     * This method is responsible for ranking the global results based on the provided rank feature results from each shard.
+     * <p>
+     * We first start by extracting ordered feature data through a {@code List<RankFeatureDoc>}
+     * from the provided rankSearchResults, and then compute the updated score for each of the documents.
+     * Once all the scores have been computed, we sort the results, perform any pagination needed, and then call the `onFinish` consumer
+     * with the final array of {@link ScoreDoc} results.
+     *
+     * @param rankSearchResults a list of rank feature results from each shard
+     * @param rankListener      a rankListener to handle the global ranking result
+     */
+    public void rankGlobalResults(List<RankFeatureResult> rankSearchResults, ActionListener<RankFeatureDoc[]> rankListener) {
+        // extract feature data from each shard rank-feature phase result
+        RankFeatureDoc[] featureDocs = extractFeatureDocs(rankSearchResults);
+
+        // generate the final `topResults` paginated results, and pass them to fetch phase through the `rankListener`
+        computeScores(featureDocs, rankListener.delegateFailureAndWrap((listener, scores) -> {
+            for (int i = 0; i < featureDocs.length; i++) {
+                featureDocs[i].score = scores[i];
+            }
+            listener.onResponse(featureDocs);
+        }));
+    }
+
+    /**
+     * Ranks the provided {@link RankFeatureDoc} array and paginates the results based on the `from` and `size` parameters.
+     */
+    public RankFeatureDoc[] rankAndPaginate(RankFeatureDoc[] rankFeatureDocs) {
+        Arrays.sort(rankFeatureDocs, Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed());
+        RankFeatureDoc[] topResults = new RankFeatureDoc[Math.max(0, Math.min(size, rankFeatureDocs.length - from))];
+        for (int rank = 0; rank < topResults.length; ++rank) {
+            topResults[rank] = rankFeatureDocs[from + rank];
+            topResults[rank].rank = from + rank + 1;
+        }
+        return topResults;
+    }
+
+    private RankFeatureDoc[] extractFeatureDocs(List<RankFeatureResult> rankSearchResults) {
+        List<RankFeatureDoc> docFeatures = new ArrayList<>();
+        for (RankFeatureResult rankFeatureResult : rankSearchResults) {
+            RankFeatureShardResult shardResult = rankFeatureResult.shardResult();
+            for (RankFeatureDoc rankFeatureDoc : shardResult.rankFeatureDocs) {
+                if (rankFeatureDoc.featureData != null) {
+                    docFeatures.add(rankFeatureDoc);
+                }
+            }
+        }
+        return docFeatures.toArray(new RankFeatureDoc[0]);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankShardContext.java b/server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankShardContext.java
new file mode 100644
index 0000000000000..5d3f30bce757a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/rank/context/RankFeaturePhaseRankShardContext.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank.context;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.rank.RankShardResult;
+
+/**
+ * {@link RankFeaturePhaseRankShardContext} is a base class used to execute the RankFeature phase on each shard.
+ * In this class, we can fetch the feature data for a given set of documents and pass them back to the coordinator
+ * through the {@link RankShardResult}.
+ */
+public abstract class RankFeaturePhaseRankShardContext {
+
+    protected final String field;
+
+    public RankFeaturePhaseRankShardContext(final String field) {
+        this.field = field;
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    /**
+     * This is used to fetch the feature data for a given set of documents, using the {@link  org.elasticsearch.search.fetch.FetchPhase}
+     * and the {@link org.elasticsearch.search.fetch.subphase.FetchFieldsPhase} subphase.
+     * The feature data is then stored in a {@link org.elasticsearch.search.rank.feature.RankFeatureDoc} and passed back to the coordinator.
+     */
+    @Nullable
+    public abstract RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId);
+}
diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java
new file mode 100644
index 0000000000000..8eb3f2fc8339b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank.feature;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.rank.RankDoc;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * A {@link RankDoc} that contains field data to be used later by the reranker on the coordinator node.
+ */
+public class RankFeatureDoc extends RankDoc {
+
+    // todo: update to support more than 1 fields; and not restrict to string data
+    public String featureData;
+
+    public RankFeatureDoc(int doc, float score, int shardIndex) {
+        super(doc, score, shardIndex);
+    }
+
+    public RankFeatureDoc(StreamInput in) throws IOException {
+        super(in);
+        featureData = in.readOptionalString();
+    }
+
+    public void featureData(String featureData) {
+        this.featureData = featureData;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(featureData);
+    }
+
+    @Override
+    protected boolean doEquals(RankDoc rd) {
+        RankFeatureDoc other = (RankFeatureDoc) rd;
+        return Objects.equals(this.featureData, other.featureData);
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hashCode(featureData);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureResult.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureResult.java
new file mode 100644
index 0000000000000..1e16d18cda367
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureResult.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank.feature;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.ShardSearchContextId;
+import org.elasticsearch.search.internal.ShardSearchRequest;
+
+import java.io.IOException;
+
+/**
+ * The result of a rank feature search phase.
+ * Each instance holds a {@code RankFeatureShardResult} along with the references associated with it.
+ */
+public class RankFeatureResult extends SearchPhaseResult {
+
+    private RankFeatureShardResult rankShardResult;
+
+    public RankFeatureResult() {}
+
+    public RankFeatureResult(ShardSearchContextId id, SearchShardTarget shardTarget, ShardSearchRequest request) {
+        this.contextId = id;
+        setSearchShardTarget(shardTarget);
+        setShardSearchRequest(request);
+    }
+
+    public RankFeatureResult(StreamInput in) throws IOException {
+        super(in);
+        contextId = new ShardSearchContextId(in);
+        rankShardResult = in.readOptionalWriteable(RankFeatureShardResult::new);
+        setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new));
+        setSearchShardTarget(in.readOptionalWriteable(SearchShardTarget::new));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        assert hasReferences();
+        contextId.writeTo(out);
+        out.writeOptionalWriteable(rankShardResult);
+        out.writeOptionalWriteable(getShardSearchRequest());
+        out.writeOptionalWriteable(getSearchShardTarget());
+    }
+
+    @Override
+    public RankFeatureResult rankFeatureResult() {
+        return this;
+    }
+
+    public void shardResult(RankFeatureShardResult shardResult) {
+        this.rankShardResult = shardResult;
+    }
+
+    public RankFeatureShardResult shardResult() {
+        return rankShardResult;
+    }
+
+    @Override
+    public boolean hasSearchContext() {
+        return rankShardResult != null;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java
new file mode 100644
index 0000000000000..727ed4e938cca
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank.feature;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.search.SearchContextSourcePrinter;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
+import org.elasticsearch.search.fetch.subphase.FetchFieldsContext;
+import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
+import org.elasticsearch.tasks.TaskCancelledException;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+/**
+ * The {@code RankFeatureShardPhase} executes the rank feature phase on the shard, iff there is a {@code RankBuilder} that requires it.
+ * This phase is responsible for reading field data for a set of docids. To do this, it reuses the {@code FetchPhase} to read the required
+ * fields for all requested documents using the `FetchFieldPhase` sub-phase.
+ */
+public final class RankFeatureShardPhase {
+
+    private static final Logger logger = LogManager.getLogger(RankFeatureShardPhase.class);
+
+    public static final RankFeatureShardResult EMPTY_RESULT = new RankFeatureShardResult(new RankFeatureDoc[0]);
+
+    public RankFeatureShardPhase() {}
+
+    public void prepareForFetch(SearchContext searchContext, RankFeatureShardRequest request) {
+        if (logger.isTraceEnabled()) {
+            logger.trace("{}", new SearchContextSourcePrinter(searchContext));
+        }
+
+        if (searchContext.isCancelled()) {
+            throw new TaskCancelledException("cancelled");
+        }
+
+        RankFeaturePhaseRankShardContext rankFeaturePhaseRankShardContext = shardContext(searchContext);
+        if (rankFeaturePhaseRankShardContext != null) {
+            assert rankFeaturePhaseRankShardContext.getField() != null : "field must not be null";
+            searchContext.fetchFieldsContext(
+                new FetchFieldsContext(Collections.singletonList(new FieldAndFormat(rankFeaturePhaseRankShardContext.getField(), null)))
+            );
+            searchContext.storedFieldsContext(StoredFieldsContext.fromList(Collections.singletonList(StoredFieldsContext._NONE_)));
+            searchContext.addFetchResult();
+            Arrays.sort(request.getDocIds());
+        }
+    }
+
+    public void processFetch(SearchContext searchContext) {
+        if (logger.isTraceEnabled()) {
+            logger.trace("{}", new SearchContextSourcePrinter(searchContext));
+        }
+
+        if (searchContext.isCancelled()) {
+            throw new TaskCancelledException("cancelled");
+        }
+
+        RankFeaturePhaseRankShardContext rankFeaturePhaseRankShardContext = searchContext.request().source().rankBuilder() != null
+            ? searchContext.request().source().rankBuilder().buildRankFeaturePhaseShardContext()
+            : null;
+        if (rankFeaturePhaseRankShardContext != null) {
+            // TODO: here we populate the profile part of the fetchResult as well
+            // we need to see what info we want to include on the overall profiling section. This is something that is per-shard
+            // so most likely we will still care about the `FetchFieldPhase` profiling info as we could potentially
+            // operate on `rank_window_size` instead of just `size` results, so this could be much more expensive.
+            FetchSearchResult fetchSearchResult = searchContext.fetchResult();
+            if (fetchSearchResult == null || fetchSearchResult.hits() == null) {
+                return;
+            }
+            // this cannot be null; as we have either already checked for it, or we would have thrown in
+            // FetchSearchResult#shardResult()
+            SearchHits hits = fetchSearchResult.hits();
+            RankFeatureShardResult featureRankShardResult = (RankFeatureShardResult) rankFeaturePhaseRankShardContext
+                .buildRankFeatureShardResult(hits, searchContext.shardTarget().getShardId().id());
+            // save the result in the search context
+            // need to add profiling info as well available from fetch
+            if (featureRankShardResult != null) {
+                searchContext.rankFeatureResult().shardResult(featureRankShardResult);
+            }
+        }
+    }
+
+    private RankFeaturePhaseRankShardContext shardContext(SearchContext searchContext) {
+        return searchContext.request().source() != null && searchContext.request().source().rankBuilder() != null
+            ? searchContext.request().source().rankBuilder().buildRankFeaturePhaseShardContext()
+            : null;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardRequest.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardRequest.java
new file mode 100644
index 0000000000000..d487fb63a0102
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardRequest.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank.feature;
+
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.action.search.SearchShardTask;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.internal.ShardSearchContextId;
+import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.transport.TransportRequest;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.IntStream;
+
+/**
+ * Shard level request for extracting all needed feature for a global reranker
+ */
+
+public class RankFeatureShardRequest extends TransportRequest implements IndicesRequest {
+
+    private final OriginalIndices originalIndices;
+    private final ShardSearchRequest shardSearchRequest;
+
+    private final ShardSearchContextId contextId;
+
+    private final int[] docIds;
+
+    public RankFeatureShardRequest(
+        OriginalIndices originalIndices,
+        ShardSearchContextId contextId,
+        ShardSearchRequest shardSearchRequest,
+        List<Integer> docIds
+    ) {
+        this.originalIndices = originalIndices;
+        this.shardSearchRequest = shardSearchRequest;
+        this.docIds = docIds.stream().flatMapToInt(IntStream::of).toArray();
+        this.contextId = contextId;
+    }
+
+    public RankFeatureShardRequest(StreamInput in) throws IOException {
+        super(in);
+        originalIndices = OriginalIndices.readOriginalIndices(in);
+        shardSearchRequest = in.readOptionalWriteable(ShardSearchRequest::new);
+        docIds = in.readIntArray();
+        contextId = in.readOptionalWriteable(ShardSearchContextId::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        OriginalIndices.writeOriginalIndices(originalIndices, out);
+        out.writeOptionalWriteable(shardSearchRequest);
+        out.writeIntArray(docIds);
+        out.writeOptionalWriteable(contextId);
+    }
+
+    @Override
+    public String[] indices() {
+        if (originalIndices == null) {
+            return null;
+        }
+        return originalIndices.indices();
+    }
+
+    @Override
+    public IndicesOptions indicesOptions() {
+        if (originalIndices == null) {
+            return null;
+        }
+        return originalIndices.indicesOptions();
+    }
+
+    public ShardSearchRequest getShardSearchRequest() {
+        return shardSearchRequest;
+    }
+
+    public int[] getDocIds() {
+        return docIds;
+    }
+
+    public ShardSearchContextId contextId() {
+        return contextId;
+    }
+
+    @Override
+    public SearchShardTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new SearchShardTask(id, type, action, getDescription(), parentTaskId, headers);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardResult.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardResult.java
new file mode 100644
index 0000000000000..e06b963621c60
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardResult.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank.feature;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.rank.RankShardResult;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * The result set of {@link RankFeatureDoc} docs for the shard.
+ */
+public class RankFeatureShardResult implements RankShardResult {
+
+    public final RankFeatureDoc[] rankFeatureDocs;
+
+    public RankFeatureShardResult(RankFeatureDoc[] rankFeatureDocs) {
+        this.rankFeatureDocs = Objects.requireNonNull(rankFeatureDocs);
+    }
+
+    public RankFeatureShardResult(StreamInput in) throws IOException {
+        rankFeatureDocs = in.readArray(RankFeatureDoc::new, RankFeatureDoc[]::new);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return "rank_feature_shard";
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.RANK_FEATURE_PHASE_ADDED;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeArray(rankFeatureDocs);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RankFeatureShardResult that = (RankFeatureShardResult) o;
+        return Arrays.equals(rankFeatureDocs, that.rankFeatureDocs);
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * Arrays.hashCode(rankFeatureDocs);
+    }
+
+    @Override
+    public String toString() {
+        return this.getClass().getSimpleName() + "{rankFeatureDocs=" + Arrays.toString(rankFeatureDocs) + '}';
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java
new file mode 100644
index 0000000000000..9716749562eae
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java
@@ -0,0 +1,1170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TotalHits;
+import org.apache.lucene.tests.store.MockDirectoryWrapper;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.SearchPhaseResult;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.internal.ShardSearchContextId;
+import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.rank.RankBuilder;
+import org.elasticsearch.search.rank.RankShardResult;
+import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
+import org.elasticsearch.search.rank.feature.RankFeatureDoc;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
+import org.elasticsearch.search.rank.feature.RankFeatureShardRequest;
+import org.elasticsearch.search.rank.feature.RankFeatureShardResult;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalAggregationTestCase;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class RankFeaturePhaseTests extends ESTestCase {
+
+    private static final int DEFAULT_RANK_WINDOW_SIZE = 10;
+    private static final int DEFAULT_FROM = 0;
+    private static final int DEFAULT_SIZE = 10;
+    private static final String DEFAULT_FIELD = "some_field";
+
+    private final RankBuilder DEFAULT_RANK_BUILDER = rankBuilder(
+        DEFAULT_RANK_WINDOW_SIZE,
+        defaultQueryPhaseRankShardContext(new ArrayList<>(), DEFAULT_RANK_WINDOW_SIZE),
+        defaultQueryPhaseRankCoordinatorContext(DEFAULT_RANK_WINDOW_SIZE),
+        defaultRankFeaturePhaseRankShardContext(DEFAULT_FIELD),
+        defaultRankFeaturePhaseRankCoordinatorContext(DEFAULT_SIZE, DEFAULT_FROM, DEFAULT_RANK_WINDOW_SIZE)
+    );
+
+    private record ExpectedRankFeatureDoc(int doc, int rank, float score, String featureData) {}
+
+    public void testRankFeaturePhaseWith1Shard() {
+        // request params used within SearchSourceBuilder and *RankContext classes
+        AtomicBoolean phaseDone = new AtomicBoolean(false);
+        final ScoreDoc[][] finalResults = new ScoreDoc[1][1];
+
+        // create a SearchSource to attach to the request
+        SearchSourceBuilder searchSourceBuilder = searchSourceWithRankBuilder(DEFAULT_RANK_BUILDER);
+
+        SearchPhaseController controller = searchPhaseController();
+        SearchShardTarget shard1Target = new SearchShardTarget("node0", new ShardId("test", "na", 0), null);
+
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
+        mockSearchPhaseContext.getRequest().source(searchSourceBuilder);
+        try (SearchPhaseResults<SearchPhaseResult> results = searchPhaseResults(controller, mockSearchPhaseContext)) {
+            final ShardSearchContextId ctx = new ShardSearchContextId(UUIDs.base64UUID(), 123);
+            QuerySearchResult queryResult = new QuerySearchResult(ctx, shard1Target, null);
+            try {
+                queryResult.setShardIndex(shard1Target.getShardId().getId());
+                // generate the QuerySearchResults that the RankFeaturePhase would have received from QueryPhase
+                // here we have 2 results, with doc ids 1 and 2
+                int totalHits = randomIntBetween(2, 100);
+                final ScoreDoc[] shard1Docs = new ScoreDoc[] { new ScoreDoc(1, 10.0F), new ScoreDoc(2, 9.0F) };
+                populateQuerySearchResult(queryResult, totalHits, shard1Docs);
+                results.consumeResult(queryResult, () -> {});
+                // do not make an actual http request, but rather generate the response
+                // as if we would have read it from the RankFeatureShardPhase
+                mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) {
+                    @Override
+                    public void sendExecuteRankFeature(
+                        Transport.Connection connection,
+                        final RankFeatureShardRequest request,
+                        SearchTask task,
+                        final SearchActionListener<RankFeatureResult> listener
+                    ) {
+                        // make sure to match the context id generated above, otherwise we throw
+                        if (request.contextId().getId() == 123 && Arrays.equals(request.getDocIds(), new int[] { 1, 2 })) {
+                            RankFeatureResult rankFeatureResult = new RankFeatureResult();
+                            buildRankFeatureResult(
+                                mockSearchPhaseContext.getRequest().source().rankBuilder(),
+                                rankFeatureResult,
+                                shard1Target,
+                                totalHits,
+                                shard1Docs
+                            );
+                            listener.onResponse(rankFeatureResult);
+                        } else {
+                            listener.onFailure(new MockDirectoryWrapper.FakeIOException());
+                        }
+                    }
+                };
+            } finally {
+                queryResult.decRef();
+            }
+
+            RankFeaturePhase rankFeaturePhase = rankFeaturePhase(results, mockSearchPhaseContext, finalResults, phaseDone);
+            try {
+                rankFeaturePhase.run();
+
+                mockSearchPhaseContext.assertNoFailure();
+                assertTrue(mockSearchPhaseContext.failures.isEmpty());
+                assertTrue(phaseDone.get());
+                assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty());
+
+                SearchPhaseResults<SearchPhaseResult> rankPhaseResults = rankFeaturePhase.rankPhaseResults;
+                assertNotNull(rankPhaseResults.getAtomicArray());
+                assertEquals(1, rankPhaseResults.getAtomicArray().length());
+                assertEquals(1, rankPhaseResults.getSuccessfulResults().count());
+
+                SearchPhaseResult shard1Result = rankPhaseResults.getAtomicArray().get(0);
+                List<ExpectedRankFeatureDoc> expectedShardResults = List.of(
+                    new ExpectedRankFeatureDoc(1, 1, 110.0F, "ranked_1"),
+                    new ExpectedRankFeatureDoc(2, 2, 109.0F, "ranked_2")
+                );
+                List<ExpectedRankFeatureDoc> expectedFinalResults = new ArrayList<>(expectedShardResults);
+                assertShardResults(shard1Result, expectedShardResults);
+                assertFinalResults(finalResults[0], expectedFinalResults);
+            } finally {
+                rankFeaturePhase.rankPhaseResults.close();
+            }
+        } finally {
+            if (mockSearchPhaseContext.searchResponse.get() != null) {
+                mockSearchPhaseContext.searchResponse.get().decRef();
+            }
+        }
+    }
+
+    public void testRankFeaturePhaseWithMultipleShardsOneEmpty() {
+        AtomicBoolean phaseDone = new AtomicBoolean(false);
+        final ScoreDoc[][] finalResults = new ScoreDoc[1][1];
+
+        // create a SearchSource to attach to the request
+        SearchSourceBuilder searchSourceBuilder = searchSourceWithRankBuilder(DEFAULT_RANK_BUILDER);
+
+        SearchPhaseController controller = searchPhaseController();
+        SearchShardTarget shard1Target = new SearchShardTarget("node0", new ShardId("test", "na", 0), null);
+        SearchShardTarget shard2Target = new SearchShardTarget("node1", new ShardId("test", "na", 1), null);
+        SearchShardTarget shard3Target = new SearchShardTarget("node2", new ShardId("test", "na", 2), null);
+
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(3);
+        mockSearchPhaseContext.getRequest().source(searchSourceBuilder);
+        try (SearchPhaseResults<SearchPhaseResult> results = searchPhaseResults(controller, mockSearchPhaseContext)) {
+            // generate the QuerySearchResults that the RankFeaturePhase would have received from QueryPhase
+            // here we have 2 results, with doc ids 1 and 2 found on shards 0 and 1 respectively
+            final ShardSearchContextId ctxShard1 = new ShardSearchContextId(UUIDs.base64UUID(), 123);
+            final ShardSearchContextId ctxShard2 = new ShardSearchContextId(UUIDs.base64UUID(), 456);
+            final ShardSearchContextId ctxShard3 = new ShardSearchContextId(UUIDs.base64UUID(), 789);
+
+            QuerySearchResult queryResultShard1 = new QuerySearchResult(ctxShard1, shard1Target, null);
+            QuerySearchResult queryResultShard2 = new QuerySearchResult(ctxShard2, shard2Target, null);
+            QuerySearchResult queryResultShard3 = new QuerySearchResult(ctxShard3, shard2Target, null);
+            try {
+                queryResultShard1.setShardIndex(shard1Target.getShardId().getId());
+                queryResultShard2.setShardIndex(shard2Target.getShardId().getId());
+                queryResultShard3.setShardIndex(shard3Target.getShardId().getId());
+
+                final int shard1Results = randomIntBetween(1, 100);
+                final int shard2Results = randomIntBetween(1, 100);
+                final int shard3Results = 0;
+
+                final ScoreDoc[] shard1Docs = new ScoreDoc[] { new ScoreDoc(1, 10.0F) };
+                populateQuerySearchResult(queryResultShard1, shard1Results, shard1Docs);
+                final ScoreDoc[] shard2Docs = new ScoreDoc[] { new ScoreDoc(2, 9.0F) };
+                populateQuerySearchResult(queryResultShard2, shard2Results, shard2Docs);
+                final ScoreDoc[] shard3Docs = new ScoreDoc[0];
+                populateQuerySearchResult(queryResultShard3, shard3Results, shard3Docs);
+
+                results.consumeResult(queryResultShard2, () -> {});
+                results.consumeResult(queryResultShard3, () -> {});
+                results.consumeResult(queryResultShard1, () -> {});
+
+                // do not make an actual http request, but rather generate the response
+                // as if we would have read it from the RankFeatureShardPhase
+                mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) {
+                    @Override
+                    public void sendExecuteRankFeature(
+                        Transport.Connection connection,
+                        final RankFeatureShardRequest request,
+                        SearchTask task,
+                        final SearchActionListener<RankFeatureResult> listener
+                    ) {
+                        // make sure to match the context id generated above, otherwise we throw
+                        // first shard
+                        RankFeatureResult rankFeatureResult = new RankFeatureResult();
+                        if (request.contextId().getId() == 123 && Arrays.equals(request.getDocIds(), new int[] { 1 })) {
+                            buildRankFeatureResult(
+                                mockSearchPhaseContext.getRequest().source().rankBuilder(),
+                                rankFeatureResult,
+                                shard1Target,
+                                shard1Results,
+                                shard1Docs
+                            );
+                            listener.onResponse(rankFeatureResult);
+                        } else if (request.contextId().getId() == 456 && Arrays.equals(request.getDocIds(), new int[] { 2 })) {
+                            // second shard
+                            buildRankFeatureResult(
+                                mockSearchPhaseContext.getRequest().source().rankBuilder(),
+                                rankFeatureResult,
+                                shard2Target,
+                                shard2Results,
+                                shard2Docs
+                            );
+                            listener.onResponse(rankFeatureResult);
+                        } else if (request.contextId().getId() == 789) {
+                            listener.onResponse(rankFeatureResult);
+                        } else {
+                            listener.onFailure(new MockDirectoryWrapper.FakeIOException());
+                        }
+                    }
+                };
+            } finally {
+                queryResultShard1.decRef();
+                queryResultShard2.decRef();
+                queryResultShard3.decRef();
+            }
+            RankFeaturePhase rankFeaturePhase = rankFeaturePhase(results, mockSearchPhaseContext, finalResults, phaseDone);
+            try {
+                rankFeaturePhase.run();
+                mockSearchPhaseContext.assertNoFailure();
+                assertTrue(mockSearchPhaseContext.failures.isEmpty());
+                assertTrue(phaseDone.get());
+                SearchPhaseResults<SearchPhaseResult> rankPhaseResults = rankFeaturePhase.rankPhaseResults;
+                assertNotNull(rankPhaseResults.getAtomicArray());
+                assertEquals(3, rankPhaseResults.getAtomicArray().length());
+                // one result is null
+                assertEquals(2, rankPhaseResults.getSuccessfulResults().count());
+
+                SearchPhaseResult shard1Result = rankPhaseResults.getAtomicArray().get(0);
+                List<ExpectedRankFeatureDoc> expectedShard1Results = List.of(new ExpectedRankFeatureDoc(1, 1, 110.0F, "ranked_1"));
+                assertShardResults(shard1Result, expectedShard1Results);
+
+                SearchPhaseResult shard2Result = rankPhaseResults.getAtomicArray().get(1);
+                List<ExpectedRankFeatureDoc> expectedShard2Results = List.of(new ExpectedRankFeatureDoc(2, 1, 109.0F, "ranked_2"));
+                assertShardResults(shard2Result, expectedShard2Results);
+
+                SearchPhaseResult shard3Result = rankPhaseResults.getAtomicArray().get(2);
+                assertNull(shard3Result);
+
+                List<ExpectedRankFeatureDoc> expectedFinalResults = List.of(
+                    new ExpectedRankFeatureDoc(1, 1, 110.0F, "ranked_1"),
+                    new ExpectedRankFeatureDoc(2, 2, 109.0F, "ranked_2")
+                );
+                assertFinalResults(finalResults[0], expectedFinalResults);
+            } finally {
+                rankFeaturePhase.rankPhaseResults.close();
+            }
+        } finally {
+            if (mockSearchPhaseContext.searchResponse.get() != null) {
+                mockSearchPhaseContext.searchResponse.get().decRef();
+            }
+        }
+    }
+
+    public void testRankFeaturePhaseNoNeedForFetchingFieldData() {
+        AtomicBoolean phaseDone = new AtomicBoolean(false);
+        final ScoreDoc[][] finalResults = new ScoreDoc[1][1];
+
+        // build the appropriate RankBuilder; using a null rankFeaturePhaseRankShardContext
+        // and non-field based rankFeaturePhaseRankCoordinatorContext
+        RankBuilder rankBuilder = rankBuilder(
+            DEFAULT_RANK_WINDOW_SIZE,
+            defaultQueryPhaseRankShardContext(Collections.emptyList(), DEFAULT_RANK_WINDOW_SIZE),
+            negatingScoresQueryFeaturePhaseRankCoordinatorContext(DEFAULT_SIZE, DEFAULT_FROM, DEFAULT_RANK_WINDOW_SIZE),
+            null,
+            null
+        );
+        // create a SearchSource to attach to the request
+        SearchSourceBuilder searchSourceBuilder = searchSourceWithRankBuilder(rankBuilder);
+
+        SearchPhaseController controller = searchPhaseController();
+        SearchShardTarget shard1Target = new SearchShardTarget("node0", new ShardId("test", "na", 0), null);
+
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
+        mockSearchPhaseContext.getRequest().source(searchSourceBuilder);
+        try (SearchPhaseResults<SearchPhaseResult> results = searchPhaseResults(controller, mockSearchPhaseContext)) {
+            // generate the QuerySearchResults that the RankFeaturePhase would have received from QueryPhase
+            // here we have 2 results, with doc ids 1 and 2
+            final ShardSearchContextId ctx = new ShardSearchContextId(UUIDs.base64UUID(), 123);
+            QuerySearchResult queryResult = new QuerySearchResult(ctx, shard1Target, null);
+
+            try {
+                queryResult.setShardIndex(shard1Target.getShardId().getId());
+                int totalHits = randomIntBetween(2, 100);
+                final ScoreDoc[] shard1Docs = new ScoreDoc[] { new ScoreDoc(1, 10.0F), new ScoreDoc(2, 9.0F) };
+                populateQuerySearchResult(queryResult, totalHits, shard1Docs);
+                results.consumeResult(queryResult, () -> {});
+                // do not make an actual http request, but rather generate the response
+                // as if we would have read it from the RankFeatureShardPhase
+                mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) {
+                    @Override
+                    public void sendExecuteRankFeature(
+                        Transport.Connection connection,
+                        final RankFeatureShardRequest request,
+                        SearchTask task,
+                        final SearchActionListener<RankFeatureResult> listener
+                    ) {
+                        // make sure to match the context id generated above, otherwise we throw
+                        if (request.contextId().getId() == 123 && Arrays.equals(request.getDocIds(), new int[] { 1, 2 })) {
+                            listener.onFailure(new UnsupportedOperationException("should not have reached here"));
+                        } else {
+                            listener.onFailure(new MockDirectoryWrapper.FakeIOException());
+                        }
+                    }
+                };
+            } finally {
+                queryResult.decRef();
+            }
+            // override the RankFeaturePhase to skip moving to next phase
+            RankFeaturePhase rankFeaturePhase = rankFeaturePhase(results, mockSearchPhaseContext, finalResults, phaseDone);
+            try {
+                rankFeaturePhase.run();
+                mockSearchPhaseContext.assertNoFailure();
+                assertTrue(mockSearchPhaseContext.failures.isEmpty());
+                assertTrue(phaseDone.get());
+
+                // in this case there was no additional "RankFeature" results on shards, so we shortcut directly to queryPhaseResults
+                SearchPhaseResults<SearchPhaseResult> rankPhaseResults = rankFeaturePhase.queryPhaseResults;
+                assertNotNull(rankPhaseResults.getAtomicArray());
+                assertEquals(1, rankPhaseResults.getAtomicArray().length());
+                assertEquals(1, rankPhaseResults.getSuccessfulResults().count());
+
+                SearchPhaseResult shardResult = rankPhaseResults.getAtomicArray().get(0);
+                assertTrue(shardResult instanceof QuerySearchResult);
+                QuerySearchResult rankResult = (QuerySearchResult) shardResult;
+                assertNull(rankResult.rankFeatureResult());
+                assertNotNull(rankResult.queryResult());
+
+                List<ExpectedRankFeatureDoc> expectedFinalResults = List.of(
+                    new ExpectedRankFeatureDoc(2, 1, -9.0F, null),
+                    new ExpectedRankFeatureDoc(1, 2, -10.0F, null)
+                );
+                assertFinalResults(finalResults[0], expectedFinalResults);
+            } finally {
+                rankFeaturePhase.rankPhaseResults.close();
+            }
+        } finally {
+            if (mockSearchPhaseContext.searchResponse.get() != null) {
+                mockSearchPhaseContext.searchResponse.get().decRef();
+            }
+        }
+    }
+
+    public void testRankFeaturePhaseOneShardFails() {
+        AtomicBoolean phaseDone = new AtomicBoolean(false);
+        final ScoreDoc[][] finalResults = new ScoreDoc[1][1];
+
+        // create a SearchSource to attach to the request
+        SearchSourceBuilder searchSourceBuilder = searchSourceWithRankBuilder(DEFAULT_RANK_BUILDER);
+
+        SearchPhaseController controller = searchPhaseController();
+        SearchShardTarget shard1Target = new SearchShardTarget("node0", new ShardId("test", "na", 0), null);
+        SearchShardTarget shard2Target = new SearchShardTarget("node1", new ShardId("test", "na", 1), null);
+
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
+        mockSearchPhaseContext.getRequest().source(searchSourceBuilder);
+        try (SearchPhaseResults<SearchPhaseResult> results = searchPhaseResults(controller, mockSearchPhaseContext)) {
+            // generate the QuerySearchResults that the RankFeaturePhase would have received from QueryPhase
+            // here we have 2 results, with doc ids 1 and 2 found on shards 0 and 1 respectively
+            final ShardSearchContextId ctxShard1 = new ShardSearchContextId(UUIDs.base64UUID(), 123);
+            final ShardSearchContextId ctxShard2 = new ShardSearchContextId(UUIDs.base64UUID(), 456);
+
+            QuerySearchResult queryResultShard1 = new QuerySearchResult(ctxShard1, shard1Target, null);
+            QuerySearchResult queryResultShard2 = new QuerySearchResult(ctxShard2, shard2Target, null);
+            try {
+                queryResultShard1.setShardIndex(shard1Target.getShardId().getId());
+                queryResultShard2.setShardIndex(shard2Target.getShardId().getId());
+
+                final int shard1Results = randomIntBetween(1, 100);
+                final ScoreDoc[] shard1Docs = new ScoreDoc[] { new ScoreDoc(1, 10.0F) };
+                populateQuerySearchResult(queryResultShard1, shard1Results, shard1Docs);
+
+                final int shard2Results = randomIntBetween(1, 100);
+                final ScoreDoc[] shard2Docs = new ScoreDoc[] { new ScoreDoc(2, 9.0F) };
+                populateQuerySearchResult(queryResultShard2, shard2Results, shard2Docs);
+
+                results.consumeResult(queryResultShard2, () -> {});
+                results.consumeResult(queryResultShard1, () -> {});
+
+                // do not make an actual http request, but rather generate the response
+                // as if we would have read it from the RankFeatureShardPhase
+                mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) {
+                    @Override
+                    public void sendExecuteRankFeature(
+                        Transport.Connection connection,
+                        final RankFeatureShardRequest request,
+                        SearchTask task,
+                        final SearchActionListener<RankFeatureResult> listener
+                    ) {
+                        // make sure to match the context id generated above, otherwise we throw
+                        // first shard
+                        if (request.contextId().getId() == 456 && Arrays.equals(request.getDocIds(), new int[] { 2 })) {
+                            RankFeatureResult rankFeatureResult = new RankFeatureResult();
+                            buildRankFeatureResult(
+                                mockSearchPhaseContext.getRequest().source().rankBuilder(),
+                                rankFeatureResult,
+                                shard2Target,
+                                shard2Results,
+                                shard2Docs
+                            );
+                            listener.onResponse(rankFeatureResult);
+
+                        } else if (request.contextId().getId() == 123 && Arrays.equals(request.getDocIds(), new int[] { 1 })) {
+                            // other shard; this one throws an exception
+                            listener.onFailure(new IllegalArgumentException("simulated failure"));
+                        } else {
+                            listener.onFailure(new MockDirectoryWrapper.FakeIOException());
+                        }
+                    }
+                };
+            } finally {
+                queryResultShard1.decRef();
+                queryResultShard2.decRef();
+            }
+            RankFeaturePhase rankFeaturePhase = rankFeaturePhase(results, mockSearchPhaseContext, finalResults, phaseDone);
+            try {
+                rankFeaturePhase.run();
+
+                mockSearchPhaseContext.assertNoFailure();
+                assertEquals(1, mockSearchPhaseContext.failures.size());
+                assertTrue(mockSearchPhaseContext.failures.get(0).getCause().getMessage().contains("simulated failure"));
+                assertTrue(phaseDone.get());
+
+                SearchPhaseResults<SearchPhaseResult> rankPhaseResults = rankFeaturePhase.rankPhaseResults;
+                assertNotNull(rankPhaseResults.getAtomicArray());
+                assertEquals(2, rankPhaseResults.getAtomicArray().length());
+                // one shard failed
+                assertEquals(1, rankPhaseResults.getSuccessfulResults().count());
+
+                SearchPhaseResult shard1Result = rankPhaseResults.getAtomicArray().get(0);
+                assertNull(shard1Result);
+
+                SearchPhaseResult shard2Result = rankPhaseResults.getAtomicArray().get(1);
+                List<ExpectedRankFeatureDoc> expectedShard2Results = List.of(new ExpectedRankFeatureDoc(2, 1, 109.0F, "ranked_2"));
+                List<ExpectedRankFeatureDoc> expectedFinalResults = new ArrayList<>(expectedShard2Results);
+                assertShardResults(shard2Result, expectedShard2Results);
+                assertFinalResults(finalResults[0], expectedFinalResults);
+            } finally {
+                rankFeaturePhase.rankPhaseResults.close();
+            }
+        } finally {
+            if (mockSearchPhaseContext.searchResponse.get() != null) {
+                mockSearchPhaseContext.searchResponse.get().decRef();
+            }
+        }
+    }
+
+    public void testRankFeaturePhaseExceptionThrownOnPhase() {
+        AtomicBoolean phaseDone = new AtomicBoolean(false);
+        final ScoreDoc[][] finalResults = new ScoreDoc[1][1];
+
+        // create a SearchSource to attach to the request
+        SearchSourceBuilder searchSourceBuilder = searchSourceWithRankBuilder(DEFAULT_RANK_BUILDER);
+
+        SearchPhaseController controller = searchPhaseController();
+        SearchShardTarget shard1Target = new SearchShardTarget("node0", new ShardId("test", "na", 0), null);
+
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
+        mockSearchPhaseContext.getRequest().source(searchSourceBuilder);
+        try (SearchPhaseResults<SearchPhaseResult> results = searchPhaseResults(controller, mockSearchPhaseContext)) {
+            // generate the QuerySearchResults that the RankFeaturePhase would have received from QueryPhase
+            // here we have 2 results, with doc ids 1 and 2
+            final ShardSearchContextId ctx = new ShardSearchContextId(UUIDs.base64UUID(), 123);
+            QuerySearchResult queryResult = new QuerySearchResult(ctx, shard1Target, null);
+            try {
+                queryResult.setShardIndex(shard1Target.getShardId().getId());
+                int totalHits = randomIntBetween(2, 100);
+                final ScoreDoc[] shard1Docs = new ScoreDoc[] { new ScoreDoc(1, 10.0F), new ScoreDoc(2, 9.0F) };
+                populateQuerySearchResult(queryResult, totalHits, shard1Docs);
+                results.consumeResult(queryResult, () -> {});
+
+                // do not make an actual http request, but rather generate the response
+                // as if we would have read it from the RankFeatureShardPhase
+                mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) {
+                    @Override
+                    public void sendExecuteRankFeature(
+                        Transport.Connection connection,
+                        final RankFeatureShardRequest request,
+                        SearchTask task,
+                        final SearchActionListener<RankFeatureResult> listener
+                    ) {
+                        // make sure to match the context id generated above, otherwise we throw
+                        if (request.contextId().getId() == 123 && Arrays.equals(request.getDocIds(), new int[] { 1, 2 })) {
+                            RankFeatureResult rankFeatureResult = new RankFeatureResult();
+                            buildRankFeatureResult(
+                                mockSearchPhaseContext.getRequest().source().rankBuilder(),
+                                rankFeatureResult,
+                                shard1Target,
+                                totalHits,
+                                shard1Docs
+                            );
+                            listener.onResponse(rankFeatureResult);
+                        } else {
+                            listener.onFailure(new MockDirectoryWrapper.FakeIOException());
+                        }
+                    }
+                };
+            } finally {
+                queryResult.decRef();
+            }
+            // override the RankFeaturePhase to raise an exception
+            RankFeaturePhase rankFeaturePhase = new RankFeaturePhase(results, null, mockSearchPhaseContext) {
+                @Override
+                void innerRun() {
+                    throw new IllegalArgumentException("simulated failure");
+                }
+
+                @Override
+                public void moveToNextPhase(
+                    SearchPhaseResults<SearchPhaseResult> phaseResults,
+                    SearchPhaseController.ReducedQueryPhase reducedQueryPhase
+                ) {
+                    // this is called after the RankFeaturePhaseCoordinatorContext has been executed
+                    phaseDone.set(true);
+                    finalResults[0] = reducedQueryPhase.sortedTopDocs().scoreDocs();
+                    logger.debug("Skipping moving to next phase");
+                }
+            };
+            assertEquals("rank-feature", rankFeaturePhase.getName());
+            try {
+                rankFeaturePhase.run();
+                assertNotNull(mockSearchPhaseContext.phaseFailure.get());
+                assertTrue(mockSearchPhaseContext.phaseFailure.get().getMessage().contains("simulated failure"));
+                assertTrue(mockSearchPhaseContext.failures.isEmpty());
+                assertFalse(phaseDone.get());
+                assertTrue(rankFeaturePhase.rankPhaseResults.getAtomicArray().asList().isEmpty());
+                assertNull(finalResults[0][0]);
+            } finally {
+                rankFeaturePhase.rankPhaseResults.close();
+            }
+        } finally {
+            if (mockSearchPhaseContext.searchResponse.get() != null) {
+                mockSearchPhaseContext.searchResponse.get().decRef();
+            }
+        }
+    }
+
+    public void testRankFeatureWithPagination() {
+        // request params used within SearchSourceBuilder and *RankContext classes
+        final int from = 1;
+        final int size = 1;
+        AtomicBoolean phaseDone = new AtomicBoolean(false);
+        final ScoreDoc[][] finalResults = new ScoreDoc[1][1];
+
+        // build the appropriate RankBuilder
+        RankBuilder rankBuilder = rankBuilder(
+            DEFAULT_RANK_WINDOW_SIZE,
+            defaultQueryPhaseRankShardContext(Collections.emptyList(), DEFAULT_RANK_WINDOW_SIZE),
+            defaultQueryPhaseRankCoordinatorContext(DEFAULT_RANK_WINDOW_SIZE),
+            defaultRankFeaturePhaseRankShardContext(DEFAULT_FIELD),
+            defaultRankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE)
+        );
+        // create a SearchSource to attach to the request
+        SearchSourceBuilder searchSourceBuilder = searchSourceWithRankBuilder(rankBuilder);
+
+        SearchPhaseController controller = searchPhaseController();
+        SearchShardTarget shard1Target = new SearchShardTarget("node0", new ShardId("test", "na", 0), null);
+        SearchShardTarget shard2Target = new SearchShardTarget("node1", new ShardId("test", "na", 1), null);
+        SearchShardTarget shard3Target = new SearchShardTarget("node2", new ShardId("test", "na", 2), null);
+
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(3);
+        mockSearchPhaseContext.getRequest().source(searchSourceBuilder);
+        try (SearchPhaseResults<SearchPhaseResult> results = searchPhaseResults(controller, mockSearchPhaseContext)) {
+            // generate the QuerySearchResults that the RankFeaturePhase would have received from QueryPhase
+            // here we have 4 results, with doc ids 1 and (11, 2, 200) found on shards 0 and 1 respectively
+            final ShardSearchContextId ctxShard1 = new ShardSearchContextId(UUIDs.base64UUID(), 123);
+            final ShardSearchContextId ctxShard2 = new ShardSearchContextId(UUIDs.base64UUID(), 456);
+            final ShardSearchContextId ctxShard3 = new ShardSearchContextId(UUIDs.base64UUID(), 789);
+
+            QuerySearchResult queryResultShard1 = new QuerySearchResult(ctxShard1, shard1Target, null);
+            QuerySearchResult queryResultShard2 = new QuerySearchResult(ctxShard2, shard2Target, null);
+            QuerySearchResult queryResultShard3 = new QuerySearchResult(ctxShard3, shard2Target, null);
+
+            try {
+                queryResultShard1.setShardIndex(shard1Target.getShardId().getId());
+                queryResultShard2.setShardIndex(shard2Target.getShardId().getId());
+                queryResultShard3.setShardIndex(shard3Target.getShardId().getId());
+
+                final int shard1Results = randomIntBetween(1, 100);
+                final ScoreDoc[] shard1Docs = new ScoreDoc[] { new ScoreDoc(1, 10.0F) };
+                populateQuerySearchResult(queryResultShard1, shard1Results, shard1Docs);
+
+                final int shard2Results = randomIntBetween(1, 100);
+                final ScoreDoc[] shard2Docs = new ScoreDoc[] {
+                    new ScoreDoc(11, 100.0F, -1),
+                    new ScoreDoc(2, 9.0F),
+                    new ScoreDoc(200, 1F, -1) };
+                populateQuerySearchResult(queryResultShard2, shard2Results, shard2Docs);
+
+                final int shard3Results = 0;
+                final ScoreDoc[] shard3Docs = new ScoreDoc[0];
+                populateQuerySearchResult(queryResultShard3, shard3Results, shard3Docs);
+
+                results.consumeResult(queryResultShard2, () -> {});
+                results.consumeResult(queryResultShard3, () -> {});
+                results.consumeResult(queryResultShard1, () -> {});
+
+                // do not make an actual http request, but rather generate the response
+                // as if we would have read it from the RankFeatureShardPhase
+                mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) {
+                    @Override
+                    public void sendExecuteRankFeature(
+                        Transport.Connection connection,
+                        final RankFeatureShardRequest request,
+                        SearchTask task,
+                        final SearchActionListener<RankFeatureResult> listener
+                    ) {
+
+                        RankFeatureResult rankFeatureResult = new RankFeatureResult();
+                        // make sure to match the context id generated above, otherwise we throw
+                        // first shard
+                        if (request.contextId().getId() == 123 && Arrays.equals(request.getDocIds(), new int[] { 1 })) {
+                            buildRankFeatureResult(
+                                mockSearchPhaseContext.getRequest().source().rankBuilder(),
+                                rankFeatureResult,
+                                shard1Target,
+                                shard1Results,
+                                shard1Docs
+                            );
+                            listener.onResponse(rankFeatureResult);
+                        } else if (request.contextId().getId() == 456 && Arrays.equals(request.getDocIds(), new int[] { 11, 2, 200 })) {
+                            // second shard
+
+                            buildRankFeatureResult(
+                                mockSearchPhaseContext.getRequest().source().rankBuilder(),
+                                rankFeatureResult,
+                                shard2Target,
+                                shard2Results,
+                                shard2Docs
+                            );
+                            listener.onResponse(rankFeatureResult);
+                        } else {
+                            listener.onFailure(new MockDirectoryWrapper.FakeIOException());
+                        }
+
+                    }
+                };
+            } finally {
+                queryResultShard1.decRef();
+                queryResultShard2.decRef();
+                queryResultShard3.decRef();
+            }
+            RankFeaturePhase rankFeaturePhase = rankFeaturePhase(results, mockSearchPhaseContext, finalResults, phaseDone);
+            try {
+                rankFeaturePhase.run();
+
+                mockSearchPhaseContext.assertNoFailure();
+                assertTrue(mockSearchPhaseContext.failures.isEmpty());
+                assertTrue(phaseDone.get());
+                SearchPhaseResults<SearchPhaseResult> rankPhaseResults = rankFeaturePhase.rankPhaseResults;
+                assertNotNull(rankPhaseResults.getAtomicArray());
+                assertEquals(3, rankPhaseResults.getAtomicArray().length());
+                // one result is null
+                assertEquals(2, rankPhaseResults.getSuccessfulResults().count());
+
+                SearchPhaseResult shard1Result = rankPhaseResults.getAtomicArray().get(0);
+                List<ExpectedRankFeatureDoc> expectedShard1Results = List.of(new ExpectedRankFeatureDoc(1, 1, 110.0F, "ranked_1"));
+                assertShardResults(shard1Result, expectedShard1Results);
+
+                SearchPhaseResult shard2Result = rankPhaseResults.getAtomicArray().get(1);
+                List<ExpectedRankFeatureDoc> expectedShard2Results = List.of(
+                    new ExpectedRankFeatureDoc(11, 1, 200.0F, "ranked_11"),
+                    new ExpectedRankFeatureDoc(2, 2, 109.0F, "ranked_2"),
+                    new ExpectedRankFeatureDoc(200, 3, 101.0F, "ranked_200")
+
+                );
+                assertShardResults(shard2Result, expectedShard2Results);
+
+                SearchPhaseResult shard3Result = rankPhaseResults.getAtomicArray().get(2);
+                assertNull(shard3Result);
+
+                List<ExpectedRankFeatureDoc> expectedFinalResults = List.of(new ExpectedRankFeatureDoc(1, 2, 110.0F, "ranked_1"));
+                assertFinalResults(finalResults[0], expectedFinalResults);
+            } finally {
+                rankFeaturePhase.rankPhaseResults.close();
+            }
+        } finally {
+            if (mockSearchPhaseContext.searchResponse.get() != null) {
+                mockSearchPhaseContext.searchResponse.get().decRef();
+            }
+        }
+    }
+
+    public void testRankFeatureCollectOnlyRankWindowSizeFeatures() {
+        // request params used within SearchSourceBuilder and *RankContext classes
+        final int rankWindowSize = 2;
+        AtomicBoolean phaseDone = new AtomicBoolean(false);
+        final ScoreDoc[][] finalResults = new ScoreDoc[1][1];
+
+        // build the appropriate RankBuilder
+        RankBuilder rankBuilder = rankBuilder(
+            rankWindowSize,
+            defaultQueryPhaseRankShardContext(Collections.emptyList(), rankWindowSize),
+            defaultQueryPhaseRankCoordinatorContext(rankWindowSize),
+            defaultRankFeaturePhaseRankShardContext(DEFAULT_FIELD),
+            defaultRankFeaturePhaseRankCoordinatorContext(DEFAULT_SIZE, DEFAULT_FROM, rankWindowSize)
+        );
+        // create a SearchSource to attach to the request
+        SearchSourceBuilder searchSourceBuilder = searchSourceWithRankBuilder(rankBuilder);
+
+        SearchPhaseController controller = searchPhaseController();
+        SearchShardTarget shard1Target = new SearchShardTarget("node0", new ShardId("test", "na", 0), null);
+        SearchShardTarget shard2Target = new SearchShardTarget("node1", new ShardId("test", "na", 1), null);
+        SearchShardTarget shard3Target = new SearchShardTarget("node2", new ShardId("test", "na", 2), null);
+
+        MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(3);
+        mockSearchPhaseContext.getRequest().source(searchSourceBuilder);
+        try (SearchPhaseResults<SearchPhaseResult> results = searchPhaseResults(controller, mockSearchPhaseContext)) {
+            // generate the QuerySearchResults that the RankFeaturePhase would have received from QueryPhase
+            // here we have 3 results, with doc ids 1, and (11, 2) found on shards 0 and 1 respectively
+            final ShardSearchContextId ctxShard1 = new ShardSearchContextId(UUIDs.base64UUID(), 123);
+            final ShardSearchContextId ctxShard2 = new ShardSearchContextId(UUIDs.base64UUID(), 456);
+            final ShardSearchContextId ctxShard3 = new ShardSearchContextId(UUIDs.base64UUID(), 789);
+
+            QuerySearchResult queryResultShard1 = new QuerySearchResult(ctxShard1, shard1Target, null);
+            QuerySearchResult queryResultShard2 = new QuerySearchResult(ctxShard2, shard2Target, null);
+            QuerySearchResult queryResultShard3 = new QuerySearchResult(ctxShard3, shard2Target, null);
+
+            try {
+                queryResultShard1.setShardIndex(shard1Target.getShardId().getId());
+                queryResultShard2.setShardIndex(shard2Target.getShardId().getId());
+                queryResultShard3.setShardIndex(shard3Target.getShardId().getId());
+
+                final int shard1Results = randomIntBetween(1, 100);
+                final ScoreDoc[] shard1Docs = new ScoreDoc[] { new ScoreDoc(1, 10.0F) };
+                populateQuerySearchResult(queryResultShard1, shard1Results, shard1Docs);
+
+                final int shard2Results = randomIntBetween(1, 100);
+                final ScoreDoc[] shard2Docs = new ScoreDoc[] { new ScoreDoc(11, 100.0F), new ScoreDoc(2, 9.0F) };
+                populateQuerySearchResult(queryResultShard2, shard2Results, shard2Docs);
+
+                final int shard3Results = 0;
+                final ScoreDoc[] shard3Docs = new ScoreDoc[0];
+                populateQuerySearchResult(queryResultShard3, shard3Results, shard3Docs);
+
+                results.consumeResult(queryResultShard2, () -> {});
+                results.consumeResult(queryResultShard3, () -> {});
+                results.consumeResult(queryResultShard1, () -> {});
+
+                // do not make an actual http request, but rather generate the response
+                // as if we would have read it from the RankFeatureShardPhase
+                mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) {
+                    @Override
+                    public void sendExecuteRankFeature(
+                        Transport.Connection connection,
+                        final RankFeatureShardRequest request,
+                        SearchTask task,
+                        final SearchActionListener<RankFeatureResult> listener
+                    ) {
+                        RankFeatureResult rankFeatureResult = new RankFeatureResult();
+                        // make sure to match the context id generated above, otherwise we throw
+                        // first shard
+                        if (request.contextId().getId() == 123 && Arrays.equals(request.getDocIds(), new int[] { 1 })) {
+                            buildRankFeatureResult(
+                                mockSearchPhaseContext.getRequest().source().rankBuilder(),
+                                rankFeatureResult,
+                                shard1Target,
+                                shard1Results,
+                                shard1Docs
+                            );
+                            listener.onResponse(rankFeatureResult);
+                        } else if (request.contextId().getId() == 456 && Arrays.equals(request.getDocIds(), new int[] { 11 })) {
+                            // second shard
+                            buildRankFeatureResult(
+                                mockSearchPhaseContext.getRequest().source().rankBuilder(),
+                                rankFeatureResult,
+                                shard2Target,
+                                shard2Results,
+                                new ScoreDoc[] { shard2Docs[0] }
+                            );
+                            listener.onResponse(rankFeatureResult);
+                        } else {
+                            listener.onFailure(new MockDirectoryWrapper.FakeIOException());
+                        }
+                    }
+                };
+            } finally {
+                queryResultShard1.decRef();
+                queryResultShard2.decRef();
+                queryResultShard3.decRef();
+            }
+            RankFeaturePhase rankFeaturePhase = rankFeaturePhase(results, mockSearchPhaseContext, finalResults, phaseDone);
+            try {
+                rankFeaturePhase.run();
+                mockSearchPhaseContext.assertNoFailure();
+                assertTrue(mockSearchPhaseContext.failures.isEmpty());
+                assertTrue(phaseDone.get());
+                SearchPhaseResults<SearchPhaseResult> rankPhaseResults = rankFeaturePhase.rankPhaseResults;
+                assertNotNull(rankPhaseResults.getAtomicArray());
+                assertEquals(3, rankPhaseResults.getAtomicArray().length());
+                // one result is null
+                assertEquals(2, rankPhaseResults.getSuccessfulResults().count());
+
+                SearchPhaseResult shard1Result = rankPhaseResults.getAtomicArray().get(0);
+                List<ExpectedRankFeatureDoc> expectedShardResults = List.of(new ExpectedRankFeatureDoc(1, 1, 110.0F, "ranked_1"));
+                assertShardResults(shard1Result, expectedShardResults);
+
+                SearchPhaseResult shard2Result = rankPhaseResults.getAtomicArray().get(1);
+                List<ExpectedRankFeatureDoc> expectedShard2Results = List.of(new ExpectedRankFeatureDoc(11, 1, 200.0F, "ranked_11"));
+                assertShardResults(shard2Result, expectedShard2Results);
+
+                SearchPhaseResult shard3Result = rankPhaseResults.getAtomicArray().get(2);
+                assertNull(shard3Result);
+
+                List<ExpectedRankFeatureDoc> expectedFinalResults = List.of(
+                    new ExpectedRankFeatureDoc(11, 1, 200.0F, "ranked_11"),
+                    new ExpectedRankFeatureDoc(1, 2, 110.0F, "ranked_1")
+                );
+                assertFinalResults(finalResults[0], expectedFinalResults);
+            } finally {
+                rankFeaturePhase.rankPhaseResults.close();
+            }
+        } finally {
+            if (mockSearchPhaseContext.searchResponse.get() != null) {
+                mockSearchPhaseContext.searchResponse.get().decRef();
+            }
+        }
+    }
+
+    private RankFeaturePhaseRankCoordinatorContext defaultRankFeaturePhaseRankCoordinatorContext(int size, int from, int rankWindowSize) {
+        return new RankFeaturePhaseRankCoordinatorContext(size, from, rankWindowSize) {
+
+            @Override
+            protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
+                // no-op
+                // this one is handled directly in rankGlobalResults to create a RankFeatureDoc
+                // and avoid modifying in-place the ScoreDoc's rank
+            }
+
+            @Override
+            public void rankGlobalResults(List<RankFeatureResult> rankSearchResults, ActionListener<RankFeatureDoc[]> rankListener) {
+                List<RankFeatureDoc> features = new ArrayList<>();
+                for (RankFeatureResult rankFeatureResult : rankSearchResults) {
+                    RankFeatureShardResult shardResult = rankFeatureResult.shardResult();
+                    features.addAll(Arrays.stream(shardResult.rankFeatureDocs).toList());
+                }
+                rankListener.onResponse(features.toArray(new RankFeatureDoc[0]));
+            }
+
+            @Override
+            public RankFeatureDoc[] rankAndPaginate(RankFeatureDoc[] rankFeatureDocs) {
+                Arrays.sort(rankFeatureDocs, Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed());
+                RankFeatureDoc[] topResults = new RankFeatureDoc[Math.max(0, Math.min(size, rankFeatureDocs.length - from))];
+                // perform pagination
+                for (int rank = 0; rank < topResults.length; ++rank) {
+                    RankFeatureDoc rfd = rankFeatureDocs[from + rank];
+                    topResults[rank] = new RankFeatureDoc(rfd.doc, rfd.score, rfd.shardIndex);
+                    topResults[rank].rank = from + rank + 1;
+                }
+                return topResults;
+            }
+        };
+    }
+
+    private QueryPhaseRankCoordinatorContext negatingScoresQueryFeaturePhaseRankCoordinatorContext(int size, int from, int rankWindowSize) {
+        return new QueryPhaseRankCoordinatorContext(rankWindowSize) {
+            @Override
+            public ScoreDoc[] rankQueryPhaseResults(
+                List<QuerySearchResult> rankSearchResults,
+                SearchPhaseController.TopDocsStats topDocsStats
+            ) {
+                List<ScoreDoc> docScores = new ArrayList<>();
+                for (QuerySearchResult phaseResults : rankSearchResults) {
+                    docScores.addAll(Arrays.asList(phaseResults.topDocs().topDocs.scoreDocs));
+                }
+                ScoreDoc[] sortedDocs = docScores.toArray(new ScoreDoc[0]);
+                // negating scores
+                Arrays.stream(sortedDocs).forEach(doc -> doc.score *= -1);
+
+                Arrays.sort(sortedDocs, Comparator.comparing((ScoreDoc doc) -> doc.score).reversed());
+                sortedDocs = Arrays.stream(sortedDocs).limit(rankWindowSize).toArray(ScoreDoc[]::new);
+                RankFeatureDoc[] topResults = new RankFeatureDoc[Math.max(0, Math.min(size, sortedDocs.length - from))];
+                // perform pagination
+                for (int rank = 0; rank < topResults.length; ++rank) {
+                    ScoreDoc base = sortedDocs[from + rank];
+                    topResults[rank] = new RankFeatureDoc(base.doc, base.score, base.shardIndex);
+                    topResults[rank].rank = from + rank + 1;
+                }
+                topDocsStats.fetchHits = topResults.length;
+                return topResults;
+            }
+        };
+    }
+
+    private RankFeaturePhaseRankShardContext defaultRankFeaturePhaseRankShardContext(String field) {
+        return new RankFeaturePhaseRankShardContext(field) {
+            @Override
+            public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) {
+                RankFeatureDoc[] rankFeatureDocs = new RankFeatureDoc[hits.getHits().length];
+                for (int i = 0; i < hits.getHits().length; i++) {
+                    SearchHit hit = hits.getHits()[i];
+                    rankFeatureDocs[i] = new RankFeatureDoc(hit.docId(), hit.getScore(), shardId);
+                    rankFeatureDocs[i].score += 100f;
+                    rankFeatureDocs[i].featureData("ranked_" + hit.docId());
+                    rankFeatureDocs[i].rank = i + 1;
+                }
+                return new RankFeatureShardResult(rankFeatureDocs);
+            }
+        };
+    }
+
+    private QueryPhaseRankCoordinatorContext defaultQueryPhaseRankCoordinatorContext(int rankWindowSize) {
+        return new QueryPhaseRankCoordinatorContext(rankWindowSize) {
+            @Override
+            public ScoreDoc[] rankQueryPhaseResults(
+                List<QuerySearchResult> querySearchResults,
+                SearchPhaseController.TopDocsStats topDocStats
+            ) {
+                List<RankFeatureDoc> rankDocs = new ArrayList<>();
+                for (int i = 0; i < querySearchResults.size(); i++) {
+                    QuerySearchResult querySearchResult = querySearchResults.get(i);
+                    RankFeatureShardResult shardResult = (RankFeatureShardResult) querySearchResult.getRankShardResult();
+                    for (RankFeatureDoc frd : shardResult.rankFeatureDocs) {
+                        frd.shardIndex = i;
+                        rankDocs.add(frd);
+                    }
+                }
+                rankDocs.sort(Comparator.comparing((RankFeatureDoc doc) -> doc.score).reversed());
+                RankFeatureDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(RankFeatureDoc[]::new);
+                topDocStats.fetchHits = topResults.length;
+                return topResults;
+            }
+        };
+    }
+
+    private QueryPhaseRankShardContext defaultQueryPhaseRankShardContext(List<Query> queries, int rankWindowSize) {
+        return new QueryPhaseRankShardContext(queries, rankWindowSize) {
+            @Override
+            public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
+                throw new UnsupportedOperationException(
+                    "shard-level QueryPhase context should not be accessed as part of the RankFeature phase"
+                );
+            }
+        };
+    }
+
+    private SearchPhaseController searchPhaseController() {
+        return new SearchPhaseController((task, request) -> InternalAggregationTestCase.emptyReduceContextBuilder());
+    }
+
+    private RankBuilder rankBuilder(
+        int rankWindowSize,
+        QueryPhaseRankShardContext queryPhaseRankShardContext,
+        QueryPhaseRankCoordinatorContext queryPhaseRankCoordinatorContext,
+        RankFeaturePhaseRankShardContext rankFeaturePhaseRankShardContext,
+        RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext
+    ) {
+        return new RankBuilder(rankWindowSize) {
+            @Override
+            protected void doWriteTo(StreamOutput out) throws IOException {
+                // no-op
+            }
+
+            @Override
+            protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+                // no-op
+            }
+
+            @Override
+            public boolean isCompoundBuilder() {
+                return true;
+            }
+
+            @Override
+            public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
+                return queryPhaseRankShardContext;
+            }
+
+            @Override
+            public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) {
+                return queryPhaseRankCoordinatorContext;
+            }
+
+            @Override
+            public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+                return rankFeaturePhaseRankShardContext;
+            }
+
+            @Override
+            public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from) {
+                return rankFeaturePhaseRankCoordinatorContext;
+            }
+
+            @Override
+            protected boolean doEquals(RankBuilder other) {
+                return other != null && other.rankWindowSize() == rankWindowSize;
+            }
+
+            @Override
+            protected int doHashCode() {
+                return 0;
+            }
+
+            @Override
+            public String getWriteableName() {
+                return "test-rank-builder";
+            }
+
+            @Override
+            public TransportVersion getMinimalSupportedVersion() {
+                return TransportVersions.V_8_12_0;
+            }
+        };
+    }
+
+    private SearchSourceBuilder searchSourceWithRankBuilder(RankBuilder rankBuilder) {
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.rankBuilder(rankBuilder);
+        return searchSourceBuilder;
+    }
+
+    private SearchPhaseResults<SearchPhaseResult> searchPhaseResults(
+        SearchPhaseController controller,
+        MockSearchPhaseContext mockSearchPhaseContext
+    ) {
+        return controller.newSearchPhaseResults(
+            EsExecutors.DIRECT_EXECUTOR_SERVICE,
+            new NoopCircuitBreaker(CircuitBreaker.REQUEST),
+            () -> false,
+            SearchProgressListener.NOOP,
+            mockSearchPhaseContext.getRequest(),
+            mockSearchPhaseContext.numShards,
+            exc -> {}
+        );
+    }
+
+    private void buildRankFeatureResult(
+        RankBuilder shardRankBuilder,
+        RankFeatureResult rankFeatureResult,
+        SearchShardTarget shardTarget,
+        int totalHits,
+        ScoreDoc[] scoreDocs
+    ) {
+        rankFeatureResult.setSearchShardTarget(shardTarget);
+        // these are the SearchHits generated by the FetchFieldPhase processor
+        SearchHit[] searchHits = new SearchHit[scoreDocs.length];
+        float maxScore = Float.MIN_VALUE;
+        for (int i = 0; i < searchHits.length; i++) {
+            searchHits[i] = SearchHit.unpooled(scoreDocs[i].doc);
+            searchHits[i].shard(shardTarget);
+            searchHits[i].score(scoreDocs[i].score);
+            searchHits[i].setDocumentField(DEFAULT_FIELD, new DocumentField(DEFAULT_FIELD, Collections.singletonList(scoreDocs[i].doc)));
+            if (scoreDocs[i].score > maxScore) {
+                maxScore = scoreDocs[i].score;
+            }
+        }
+        SearchHits hits = null;
+        try {
+            hits = SearchHits.unpooled(searchHits, new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), maxScore);
+            // construct the appropriate RankFeatureDoc objects based on the rank builder
+            RankFeaturePhaseRankShardContext rankFeaturePhaseRankShardContext = shardRankBuilder.buildRankFeaturePhaseShardContext();
+            RankFeatureShardResult rankShardResult = (RankFeatureShardResult) rankFeaturePhaseRankShardContext.buildRankFeatureShardResult(
+                hits,
+                shardTarget.getShardId().id()
+            );
+            rankFeatureResult.shardResult(rankShardResult);
+        } finally {
+            if (hits != null) {
+                hits.decRef();
+            }
+        }
+    }
+
+    private void populateQuerySearchResult(QuerySearchResult queryResult, int totalHits, ScoreDoc[] scoreDocs) {
+        // this would have been populated during the QueryPhase by the appropriate QueryPhaseShardContext
+        float maxScore = Float.MIN_VALUE;
+        RankFeatureDoc[] rankFeatureDocs = new RankFeatureDoc[scoreDocs.length];
+        for (int i = 0; i < scoreDocs.length; i++) {
+            if (scoreDocs[i].score > maxScore) {
+                maxScore = scoreDocs[i].score;
+            }
+            rankFeatureDocs[i] = new RankFeatureDoc(scoreDocs[i].doc, scoreDocs[i].score, scoreDocs[i].shardIndex);
+        }
+        queryResult.setRankShardResult(new RankFeatureShardResult(rankFeatureDocs));
+        queryResult.topDocs(
+            new TopDocsAndMaxScore(
+                new TopDocs(new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), scoreDocs),
+                maxScore
+
+            ),
+            new DocValueFormat[0]
+        );
+        queryResult.size(totalHits);
+    }
+
+    private RankFeaturePhase rankFeaturePhase(
+        SearchPhaseResults<SearchPhaseResult> results,
+        MockSearchPhaseContext mockSearchPhaseContext,
+        ScoreDoc[][] finalResults,
+        AtomicBoolean phaseDone
+    ) {
+        // override the RankFeaturePhase to skip moving to next phase
+        return new RankFeaturePhase(results, null, mockSearchPhaseContext) {
+            @Override
+            public void moveToNextPhase(
+                SearchPhaseResults<SearchPhaseResult> phaseResults,
+                SearchPhaseController.ReducedQueryPhase reducedQueryPhase
+            ) {
+                // this is called after the RankFeaturePhaseCoordinatorContext has been executed
+                phaseDone.set(true);
+                finalResults[0] = reducedQueryPhase.sortedTopDocs().scoreDocs();
+                logger.debug("Skipping moving to next phase");
+            }
+        };
+    }
+
+    private void assertRankFeatureResults(RankFeatureShardResult rankFeatureShardResult, List<ExpectedRankFeatureDoc> expectedResults) {
+        assertEquals(expectedResults.size(), rankFeatureShardResult.rankFeatureDocs.length);
+        for (int i = 0; i < expectedResults.size(); i++) {
+            ExpectedRankFeatureDoc expected = expectedResults.get(i);
+            RankFeatureDoc actual = rankFeatureShardResult.rankFeatureDocs[i];
+            assertEquals(expected.doc, actual.doc);
+            assertEquals(expected.rank, actual.rank);
+            assertEquals(expected.score, actual.score, 10E-5);
+            assertEquals(expected.featureData, actual.featureData);
+        }
+    }
+
+    private void assertFinalResults(ScoreDoc[] finalResults, List<ExpectedRankFeatureDoc> expectedResults) {
+        assertEquals(expectedResults.size(), finalResults.length);
+        for (int i = 0; i < expectedResults.size(); i++) {
+            ExpectedRankFeatureDoc expected = expectedResults.get(i);
+            RankFeatureDoc actual = (RankFeatureDoc) finalResults[i];
+            assertEquals(expected.doc, actual.doc);
+            assertEquals(expected.rank, actual.rank);
+            assertEquals(expected.score, actual.score, 10E-5);
+        }
+    }
+
+    private void assertShardResults(SearchPhaseResult shardResult, List<ExpectedRankFeatureDoc> expectedShardResults) {
+        assertTrue(shardResult instanceof RankFeatureResult);
+        RankFeatureResult rankResult = (RankFeatureResult) shardResult;
+        assertNotNull(rankResult.rankFeatureResult());
+        assertNull(rankResult.queryResult());
+        assertNotNull(rankResult.rankFeatureResult().shardResult());
+        RankFeatureShardResult rankFeatureShardResult = rankResult.rankFeatureResult().shardResult();
+        assertRankFeatureResults(rankFeatureShardResult, expectedShardResults);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 59acb227385f6..4d58471f4817a 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -644,8 +644,8 @@ public void testIsParallelCollectionSupportedForResults() {
         ToLongFunction<String> fieldCardinality = name -> -1;
         for (var resultsType : SearchService.ResultsType.values()) {
             switch (resultsType) {
-                case NONE, FETCH -> assertFalse(
-                    "NONE and FETCH phases do not support parallel collection.",
+                case NONE, RANK_FEATURE, FETCH -> assertFalse(
+                    "NONE, RANK_FEATURE, and FETCH phases do not support parallel collection.",
                     DefaultSearchContext.isParallelCollectionSupportedForResults(
                         resultsType,
                         searchSourceBuilderOrNull,
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index d2c6c55634ec6..2af20a6ffef4a 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -13,6 +13,8 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TotalHitCountCollectorManager;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.SetOnce;
@@ -27,6 +29,7 @@
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClosePointInTimeRequest;
 import org.elasticsearch.action.search.OpenPointInTimeRequest;
+import org.elasticsearch.action.search.SearchPhaseController;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
@@ -92,6 +95,7 @@
 import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchRequest;
+import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
@@ -102,12 +106,26 @@
 import org.elasticsearch.search.query.NonCountingTermQuery;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.rank.RankBuilder;
+import org.elasticsearch.search.rank.RankShardResult;
+import org.elasticsearch.search.rank.TestRankBuilder;
+import org.elasticsearch.search.rank.TestRankDoc;
+import org.elasticsearch.search.rank.TestRankShardResult;
+import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
+import org.elasticsearch.search.rank.feature.RankFeatureDoc;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
+import org.elasticsearch.search.rank.feature.RankFeatureShardRequest;
+import org.elasticsearch.search.rank.feature.RankFeatureShardResult;
 import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.tasks.TaskCancelHelper;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -115,8 +133,10 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
@@ -136,8 +156,8 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
-import static org.elasticsearch.indices.cluster.AbstractIndicesClusterStateServiceTestCase.awaitIndexShardCloseAsyncTasks;
 import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.DELETED;
+import static org.elasticsearch.search.SearchService.DEFAULT_SIZE;
 import static org.elasticsearch.search.SearchService.QUERY_PHASE_PARALLEL_COLLECTION_ENABLED;
 import static org.elasticsearch.search.SearchService.SEARCH_WORKER_THREADS_ENABLED;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -371,7 +391,7 @@ public void testSearchWhileIndexDeleted() throws InterruptedException {
                                 -1,
                                 null
                             ),
-                            new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()),
+                            new SearchShardTask(123L, "", "", "", null, emptyMap()),
                             result.delegateFailure((l, r) -> {
                                 r.incRef();
                                 l.onResponse(r);
@@ -387,7 +407,7 @@ public void testSearchWhileIndexDeleted() throws InterruptedException {
                                 null/* not a scroll */
                             );
                             PlainActionFuture<FetchSearchResult> listener = new PlainActionFuture<>();
-                            service.executeFetchPhase(req, new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()), listener);
+                            service.executeFetchPhase(req, new SearchShardTask(123L, "", "", "", null, emptyMap()), listener);
                             listener.get();
                             if (useScroll) {
                                 // have to free context since this test does not remove the index from IndicesService.
@@ -422,6 +442,711 @@ public void testSearchWhileIndexDeleted() throws InterruptedException {
         assertEquals(0, totalStats.getFetchCurrent());
     }
 
+    public void testRankFeaturePhaseSearchPhases() throws InterruptedException, ExecutionException {
+        final String indexName = "index";
+        final String rankFeatureFieldName = "field";
+        final String searchFieldName = "search_field";
+        final String searchFieldValue = "some_value";
+        final String fetchFieldName = "fetch_field";
+        final String fetchFieldValue = "fetch_value";
+
+        final int minDocs = 3;
+        final int maxDocs = 10;
+        int numDocs = between(minDocs, maxDocs);
+        createIndex(indexName);
+        // index some documents
+        for (int i = 0; i < numDocs; i++) {
+            prepareIndex(indexName).setId(String.valueOf(i))
+                .setSource(
+                    rankFeatureFieldName,
+                    "aardvark_" + i,
+                    searchFieldName,
+                    searchFieldValue,
+                    fetchFieldName,
+                    fetchFieldValue + "_" + i
+                )
+                .get();
+        }
+        indicesAdmin().prepareRefresh(indexName).get();
+
+        final SearchService service = getInstanceFromNode(SearchService.class);
+
+        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        final IndexService indexService = indicesService.indexServiceSafe(resolveIndex(indexName));
+        final IndexShard indexShard = indexService.getShard(0);
+        SearchShardTask searchTask = new SearchShardTask(123L, "", "", "", null, emptyMap());
+
+        // create a SearchRequest that will return all documents and defines a TestRankBuilder with shard-level only operations
+        SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true)
+            .source(
+                new SearchSourceBuilder().query(new TermQueryBuilder(searchFieldName, searchFieldValue))
+                    .size(DEFAULT_SIZE)
+                    .fetchField(fetchFieldName)
+                    .rankBuilder(
+                        // here we override only the shard-level contexts
+                        new TestRankBuilder(RankBuilder.DEFAULT_RANK_WINDOW_SIZE) {
+                            @Override
+                            public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
+                                return new QueryPhaseRankShardContext(queries, from) {
+
+                                    @Override
+                                    public int rankWindowSize() {
+                                        return DEFAULT_RANK_WINDOW_SIZE;
+                                    }
+
+                                    @Override
+                                    public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
+                                        // we know we have just 1 query, so return all the docs from it
+                                        return new TestRankShardResult(
+                                            Arrays.stream(rankResults.get(0).scoreDocs)
+                                                .map(x -> new TestRankDoc(x.doc, x.score, x.shardIndex))
+                                                .limit(rankWindowSize())
+                                                .toArray(TestRankDoc[]::new)
+                                        );
+                                    }
+                                };
+                            }
+
+                            @Override
+                            public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+                                return new RankFeaturePhaseRankShardContext(rankFeatureFieldName) {
+                                    @Override
+                                    public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) {
+                                        RankFeatureDoc[] rankFeatureDocs = new RankFeatureDoc[hits.getHits().length];
+                                        for (int i = 0; i < hits.getHits().length; i++) {
+                                            SearchHit hit = hits.getHits()[i];
+                                            rankFeatureDocs[i] = new RankFeatureDoc(hit.docId(), hit.getScore(), shardId);
+                                            rankFeatureDocs[i].featureData(hit.getFields().get(rankFeatureFieldName).getValue());
+                                            rankFeatureDocs[i].score = (numDocs - i) + randomFloat();
+                                            rankFeatureDocs[i].rank = i + 1;
+                                        }
+                                        return new RankFeatureShardResult(rankFeatureDocs);
+                                    }
+                                };
+                            }
+                        }
+                    )
+            );
+
+        ShardSearchRequest request = new ShardSearchRequest(
+            OriginalIndices.NONE,
+            searchRequest,
+            indexShard.shardId(),
+            0,
+            1,
+            AliasFilter.EMPTY,
+            1.0f,
+            -1,
+            null
+        );
+        QuerySearchResult queryResult = null;
+        RankFeatureResult rankResult = null;
+        try {
+            // Execute the query phase and store the result in a SearchPhaseResult container using a PlainActionFuture
+            PlainActionFuture<SearchPhaseResult> queryPhaseResults = new PlainActionFuture<>();
+            service.executeQueryPhase(request, searchTask, queryPhaseResults);
+            queryResult = (QuerySearchResult) queryPhaseResults.get();
+
+            // these are the matched docs from the query phase
+            final TestRankDoc[] queryRankDocs = ((TestRankShardResult) queryResult.getRankShardResult()).testRankDocs;
+
+            // assume that we have cut down to these from the coordinator node as the top-docs to run the rank feature phase upon
+            List<Integer> topRankWindowSizeDocs = randomNonEmptySubsetOf(Arrays.stream(queryRankDocs).map(x -> x.doc).toList());
+
+            // now we create a RankFeatureShardRequest to extract feature info for the top-docs above
+            RankFeatureShardRequest rankFeatureShardRequest = new RankFeatureShardRequest(
+                OriginalIndices.NONE,
+                queryResult.getContextId(), // use the context from the query phase
+                request,
+                topRankWindowSizeDocs
+            );
+            PlainActionFuture<RankFeatureResult> rankPhaseResults = new PlainActionFuture<>();
+            service.executeRankFeaturePhase(rankFeatureShardRequest, searchTask, rankPhaseResults);
+            rankResult = rankPhaseResults.get();
+
+            assertNotNull(rankResult);
+            assertNotNull(rankResult.rankFeatureResult());
+            RankFeatureShardResult rankFeatureShardResult = rankResult.rankFeatureResult().shardResult();
+            assertNotNull(rankFeatureShardResult);
+
+            List<Integer> sortedRankWindowDocs = topRankWindowSizeDocs.stream().sorted().toList();
+            assertEquals(sortedRankWindowDocs.size(), rankFeatureShardResult.rankFeatureDocs.length);
+            for (int i = 0; i < sortedRankWindowDocs.size(); i++) {
+                assertEquals((long) sortedRankWindowDocs.get(i), rankFeatureShardResult.rankFeatureDocs[i].doc);
+                assertEquals(rankFeatureShardResult.rankFeatureDocs[i].featureData, "aardvark_" + sortedRankWindowDocs.get(i));
+            }
+
+            List<Integer> globalTopKResults = randomNonEmptySubsetOf(
+                Arrays.stream(rankFeatureShardResult.rankFeatureDocs).map(x -> x.doc).toList()
+            );
+
+            // finally let's create a fetch request to bring back fetch info for the top results
+            ShardFetchSearchRequest fetchRequest = new ShardFetchSearchRequest(
+                OriginalIndices.NONE,
+                rankResult.getContextId(),
+                request,
+                globalTopKResults,
+                null,
+                rankResult.getRescoreDocIds(),
+                null
+            );
+
+            // execute fetch phase and perform any validations once we retrieve the response
+            // the difference in how we do assertions here is needed because once the transport service sends back the response
+            // it decrements the reference to the FetchSearchResult (through the ActionListener#respondAndRelease) and sets hits to null
+            service.executeFetchPhase(fetchRequest, searchTask, new ActionListener<>() {
+                @Override
+                public void onResponse(FetchSearchResult fetchSearchResult) {
+                    assertNotNull(fetchSearchResult);
+                    assertNotNull(fetchSearchResult.hits());
+
+                    int totalHits = fetchSearchResult.hits().getHits().length;
+                    assertEquals(globalTopKResults.size(), totalHits);
+                    for (int i = 0; i < totalHits; i++) {
+                        // rank and score are set by the SearchPhaseController#merge so no need to validate that here
+                        SearchHit hit = fetchSearchResult.hits().getAt(i);
+                        assertNotNull(hit.getFields().get(fetchFieldName));
+                        assertEquals(hit.getFields().get(fetchFieldName).getValue(), fetchFieldValue + "_" + hit.docId());
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    throw new AssertionError("No failure should have been raised", e);
+                }
+            });
+        } catch (Exception ex) {
+            if (queryResult != null) {
+                if (queryResult.hasReferences()) {
+                    queryResult.decRef();
+                }
+                service.freeReaderContext(queryResult.getContextId());
+            }
+            if (rankResult != null && rankResult.hasReferences()) {
+                rankResult.decRef();
+            }
+            throw ex;
+        }
+    }
+
+    public void testRankFeaturePhaseUsingClient() {
+        final String indexName = "index";
+        final String rankFeatureFieldName = "field";
+        final String searchFieldName = "search_field";
+        final String searchFieldValue = "some_value";
+        final String fetchFieldName = "fetch_field";
+        final String fetchFieldValue = "fetch_value";
+
+        final int minDocs = 4;
+        final int maxDocs = 10;
+        int numDocs = between(minDocs, maxDocs);
+        createIndex(indexName);
+        // index some documents
+        for (int i = 0; i < numDocs; i++) {
+            prepareIndex(indexName).setId(String.valueOf(i))
+                .setSource(
+                    rankFeatureFieldName,
+                    "aardvark_" + i,
+                    searchFieldName,
+                    searchFieldValue,
+                    fetchFieldName,
+                    fetchFieldValue + "_" + i
+                )
+                .get();
+        }
+        indicesAdmin().prepareRefresh(indexName).get();
+
+        ElasticsearchAssertions.assertResponse(
+            client().prepareSearch(indexName)
+                .setSource(
+                    new SearchSourceBuilder().query(new TermQueryBuilder(searchFieldName, searchFieldValue))
+                        .size(2)
+                        .from(2)
+                        .fetchField(fetchFieldName)
+                        .rankBuilder(
+                            // here we override only the shard-level contexts
+                            new TestRankBuilder(RankBuilder.DEFAULT_RANK_WINDOW_SIZE) {
+
+                                // no need for more than one queries
+                                @Override
+                                public boolean isCompoundBuilder() {
+                                    return false;
+                                }
+
+                                @Override
+                                public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from) {
+                                    return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) {
+                                        @Override
+                                        protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
+                                            float[] scores = new float[featureDocs.length];
+                                            for (int i = 0; i < featureDocs.length; i++) {
+                                                scores[i] = featureDocs[i].score;
+                                            }
+                                            scoreListener.onResponse(scores);
+                                        }
+                                    };
+                                }
+
+                                @Override
+                                public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) {
+                                    return new QueryPhaseRankCoordinatorContext(RankBuilder.DEFAULT_RANK_WINDOW_SIZE) {
+                                        @Override
+                                        public ScoreDoc[] rankQueryPhaseResults(
+                                            List<QuerySearchResult> querySearchResults,
+                                            SearchPhaseController.TopDocsStats topDocStats
+                                        ) {
+                                            List<TestRankDoc> rankDocs = new ArrayList<>();
+                                            for (int i = 0; i < querySearchResults.size(); i++) {
+                                                QuerySearchResult querySearchResult = querySearchResults.get(i);
+                                                TestRankShardResult shardResult = (TestRankShardResult) querySearchResult
+                                                    .getRankShardResult();
+                                                for (TestRankDoc trd : shardResult.testRankDocs) {
+                                                    trd.shardIndex = i;
+                                                    rankDocs.add(trd);
+                                                }
+                                            }
+                                            rankDocs.sort(Comparator.comparing((TestRankDoc doc) -> doc.score).reversed());
+                                            TestRankDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(TestRankDoc[]::new);
+                                            topDocStats.fetchHits = topResults.length;
+                                            return topResults;
+                                        }
+                                    };
+                                }
+
+                                @Override
+                                public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
+                                    return new QueryPhaseRankShardContext(queries, from) {
+
+                                        @Override
+                                        public int rankWindowSize() {
+                                            return DEFAULT_RANK_WINDOW_SIZE;
+                                        }
+
+                                        @Override
+                                        public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
+                                            // we know we have just 1 query, so return all the docs from it
+                                            return new TestRankShardResult(
+                                                Arrays.stream(rankResults.get(0).scoreDocs)
+                                                    .map(x -> new TestRankDoc(x.doc, x.score, x.shardIndex))
+                                                    .limit(rankWindowSize())
+                                                    .toArray(TestRankDoc[]::new)
+                                            );
+                                        }
+                                    };
+                                }
+
+                                @Override
+                                public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+                                    return new RankFeaturePhaseRankShardContext(rankFeatureFieldName) {
+                                        @Override
+                                        public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) {
+                                            RankFeatureDoc[] rankFeatureDocs = new RankFeatureDoc[hits.getHits().length];
+                                            for (int i = 0; i < hits.getHits().length; i++) {
+                                                SearchHit hit = hits.getHits()[i];
+                                                rankFeatureDocs[i] = new RankFeatureDoc(hit.docId(), hit.getScore(), shardId);
+                                                rankFeatureDocs[i].featureData(hit.getFields().get(rankFeatureFieldName).getValue());
+                                                rankFeatureDocs[i].score = randomFloat();
+                                                rankFeatureDocs[i].rank = i + 1;
+                                            }
+                                            return new RankFeatureShardResult(rankFeatureDocs);
+                                        }
+                                    };
+                                }
+                            }
+                        )
+                ),
+            (response) -> {
+                SearchHits hits = response.getHits();
+                assertEquals(hits.getTotalHits().value, numDocs);
+                assertEquals(hits.getHits().length, 2);
+                int index = 0;
+                for (SearchHit hit : hits.getHits()) {
+                    assertEquals(hit.getRank(), 3 + index);
+                    assertTrue(hit.getScore() >= 0);
+                    assertEquals(hit.getFields().get(fetchFieldName).getValue(), fetchFieldValue + "_" + hit.docId());
+                    index++;
+                }
+            }
+        );
+    }
+
+    public void testRankFeaturePhaseExceptionOnCoordinatingNode() {
+        final String indexName = "index";
+        final String rankFeatureFieldName = "field";
+        final String searchFieldName = "search_field";
+        final String searchFieldValue = "some_value";
+        final String fetchFieldName = "fetch_field";
+        final String fetchFieldValue = "fetch_value";
+
+        final int minDocs = 3;
+        final int maxDocs = 10;
+        int numDocs = between(minDocs, maxDocs);
+        createIndex(indexName);
+        // index some documents
+        for (int i = 0; i < numDocs; i++) {
+            prepareIndex(indexName).setId(String.valueOf(i))
+                .setSource(
+                    rankFeatureFieldName,
+                    "aardvark_" + i,
+                    searchFieldName,
+                    searchFieldValue,
+                    fetchFieldName,
+                    fetchFieldValue + "_" + i
+                )
+                .get();
+        }
+        indicesAdmin().prepareRefresh(indexName).get();
+
+        expectThrows(
+            SearchPhaseExecutionException.class,
+            () -> client().prepareSearch(indexName)
+                .setSource(
+                    new SearchSourceBuilder().query(new TermQueryBuilder(searchFieldName, searchFieldValue))
+                        .size(2)
+                        .from(2)
+                        .fetchField(fetchFieldName)
+                        .rankBuilder(new TestRankBuilder(RankBuilder.DEFAULT_RANK_WINDOW_SIZE) {
+
+                            // no need for more than one queries
+                            @Override
+                            public boolean isCompoundBuilder() {
+                                return false;
+                            }
+
+                            @Override
+                            public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from) {
+                                return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) {
+                                    @Override
+                                    protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
+                                        throw new IllegalStateException("should have failed earlier");
+                                    }
+                                };
+                            }
+
+                            @Override
+                            public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) {
+                                return new QueryPhaseRankCoordinatorContext(RankBuilder.DEFAULT_RANK_WINDOW_SIZE) {
+                                    @Override
+                                    public ScoreDoc[] rankQueryPhaseResults(
+                                        List<QuerySearchResult> querySearchResults,
+                                        SearchPhaseController.TopDocsStats topDocStats
+                                    ) {
+                                        throw new UnsupportedOperationException("simulated failure");
+                                    }
+                                };
+                            }
+
+                            @Override
+                            public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
+                                return new QueryPhaseRankShardContext(queries, from) {
+
+                                    @Override
+                                    public int rankWindowSize() {
+                                        return DEFAULT_RANK_WINDOW_SIZE;
+                                    }
+
+                                    @Override
+                                    public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
+                                        // we know we have just 1 query, so return all the docs from it
+                                        return new TestRankShardResult(
+                                            Arrays.stream(rankResults.get(0).scoreDocs)
+                                                .map(x -> new TestRankDoc(x.doc, x.score, x.shardIndex))
+                                                .limit(rankWindowSize())
+                                                .toArray(TestRankDoc[]::new)
+                                        );
+                                    }
+                                };
+                            }
+
+                            @Override
+                            public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+                                return new RankFeaturePhaseRankShardContext(rankFeatureFieldName) {
+                                    @Override
+                                    public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) {
+                                        RankFeatureDoc[] rankFeatureDocs = new RankFeatureDoc[hits.getHits().length];
+                                        for (int i = 0; i < hits.getHits().length; i++) {
+                                            SearchHit hit = hits.getHits()[i];
+                                            rankFeatureDocs[i] = new RankFeatureDoc(hit.docId(), hit.getScore(), shardId);
+                                            rankFeatureDocs[i].featureData(hit.getFields().get(rankFeatureFieldName).getValue());
+                                            rankFeatureDocs[i].score = randomFloat();
+                                            rankFeatureDocs[i].rank = i + 1;
+                                        }
+                                        return new RankFeatureShardResult(rankFeatureDocs);
+                                    }
+                                };
+                            }
+                        })
+                )
+                .get()
+        );
+    }
+
+    public void testRankFeaturePhaseExceptionAllShardFail() {
+        final String indexName = "index";
+        final String rankFeatureFieldName = "field";
+        final String searchFieldName = "search_field";
+        final String searchFieldValue = "some_value";
+        final String fetchFieldName = "fetch_field";
+        final String fetchFieldValue = "fetch_value";
+
+        final int minDocs = 3;
+        final int maxDocs = 10;
+        int numDocs = between(minDocs, maxDocs);
+        createIndex(indexName);
+        // index some documents
+        for (int i = 0; i < numDocs; i++) {
+            prepareIndex(indexName).setId(String.valueOf(i))
+                .setSource(
+                    rankFeatureFieldName,
+                    "aardvark_" + i,
+                    searchFieldName,
+                    searchFieldValue,
+                    fetchFieldName,
+                    fetchFieldValue + "_" + i
+                )
+                .get();
+        }
+        indicesAdmin().prepareRefresh(indexName).get();
+
+        expectThrows(
+            SearchPhaseExecutionException.class,
+            () -> client().prepareSearch(indexName)
+                .setAllowPartialSearchResults(true)
+                .setSource(
+                    new SearchSourceBuilder().query(new TermQueryBuilder(searchFieldName, searchFieldValue))
+                        .fetchField(fetchFieldName)
+                        .rankBuilder(
+                            // here we override only the shard-level contexts
+                            new TestRankBuilder(RankBuilder.DEFAULT_RANK_WINDOW_SIZE) {
+
+                                // no need for more than one queries
+                                @Override
+                                public boolean isCompoundBuilder() {
+                                    return false;
+                                }
+
+                                @Override
+                                public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from) {
+                                    return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) {
+                                        @Override
+                                        protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
+                                            float[] scores = new float[featureDocs.length];
+                                            for (int i = 0; i < featureDocs.length; i++) {
+                                                scores[i] = featureDocs[i].score;
+                                            }
+                                            scoreListener.onResponse(scores);
+                                        }
+                                    };
+                                }
+
+                                @Override
+                                public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) {
+                                    return new QueryPhaseRankCoordinatorContext(RankBuilder.DEFAULT_RANK_WINDOW_SIZE) {
+                                        @Override
+                                        public ScoreDoc[] rankQueryPhaseResults(
+                                            List<QuerySearchResult> querySearchResults,
+                                            SearchPhaseController.TopDocsStats topDocStats
+                                        ) {
+                                            List<TestRankDoc> rankDocs = new ArrayList<>();
+                                            for (int i = 0; i < querySearchResults.size(); i++) {
+                                                QuerySearchResult querySearchResult = querySearchResults.get(i);
+                                                TestRankShardResult shardResult = (TestRankShardResult) querySearchResult
+                                                    .getRankShardResult();
+                                                for (TestRankDoc trd : shardResult.testRankDocs) {
+                                                    trd.shardIndex = i;
+                                                    rankDocs.add(trd);
+                                                }
+                                            }
+                                            rankDocs.sort(Comparator.comparing((TestRankDoc doc) -> doc.score).reversed());
+                                            TestRankDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(TestRankDoc[]::new);
+                                            topDocStats.fetchHits = topResults.length;
+                                            return topResults;
+                                        }
+                                    };
+                                }
+
+                                @Override
+                                public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
+                                    return new QueryPhaseRankShardContext(queries, from) {
+
+                                        @Override
+                                        public int rankWindowSize() {
+                                            return DEFAULT_RANK_WINDOW_SIZE;
+                                        }
+
+                                        @Override
+                                        public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
+                                            // we know we have just 1 query, so return all the docs from it
+                                            return new TestRankShardResult(
+                                                Arrays.stream(rankResults.get(0).scoreDocs)
+                                                    .map(x -> new TestRankDoc(x.doc, x.score, x.shardIndex))
+                                                    .limit(rankWindowSize())
+                                                    .toArray(TestRankDoc[]::new)
+                                            );
+                                        }
+                                    };
+                                }
+
+                                @Override
+                                public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+                                    return new RankFeaturePhaseRankShardContext(rankFeatureFieldName) {
+                                        @Override
+                                        public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) {
+                                            throw new UnsupportedOperationException("simulated failure");
+                                        }
+                                    };
+                                }
+                            }
+                        )
+                )
+                .get()
+        );
+    }
+
+    public void testRankFeaturePhaseExceptionOneShardFails() {
+        // if we have only one shard and it fails, it will fallback to context.onPhaseFailure which will eventually clean up all contexts.
+        // in this test we want to make sure that even if one shard (of many) fails during the RankFeaturePhase, then the appropriate
+        // context will have been cleaned up.
+        final String indexName = "index";
+        final String rankFeatureFieldName = "field";
+        final String searchFieldName = "search_field";
+        final String searchFieldValue = "some_value";
+        final String fetchFieldName = "fetch_field";
+        final String fetchFieldValue = "fetch_value";
+
+        final int minDocs = 3;
+        final int maxDocs = 10;
+        int numDocs = between(minDocs, maxDocs);
+        createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).build());
+        // index some documents
+        for (int i = 0; i < numDocs; i++) {
+            prepareIndex(indexName).setId(String.valueOf(i))
+                .setSource(
+                    rankFeatureFieldName,
+                    "aardvark_" + i,
+                    searchFieldName,
+                    searchFieldValue,
+                    fetchFieldName,
+                    fetchFieldValue + "_" + i
+                )
+                .get();
+        }
+        indicesAdmin().prepareRefresh(indexName).get();
+
+        assertResponse(
+            client().prepareSearch(indexName)
+                .setAllowPartialSearchResults(true)
+                .setSource(
+                    new SearchSourceBuilder().query(new TermQueryBuilder(searchFieldName, searchFieldValue))
+                        .fetchField(fetchFieldName)
+                        .rankBuilder(
+                            // here we override only the shard-level contexts
+                            new TestRankBuilder(RankBuilder.DEFAULT_RANK_WINDOW_SIZE) {
+
+                                // no need for more than one queries
+                                @Override
+                                public boolean isCompoundBuilder() {
+                                    return false;
+                                }
+
+                                @Override
+                                public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from) {
+                                    return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) {
+                                        @Override
+                                        protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener<float[]> scoreListener) {
+                                            float[] scores = new float[featureDocs.length];
+                                            for (int i = 0; i < featureDocs.length; i++) {
+                                                scores[i] = featureDocs[i].score;
+                                            }
+                                            scoreListener.onResponse(scores);
+                                        }
+                                    };
+                                }
+
+                                @Override
+                                public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) {
+                                    return new QueryPhaseRankCoordinatorContext(RankBuilder.DEFAULT_RANK_WINDOW_SIZE) {
+                                        @Override
+                                        public ScoreDoc[] rankQueryPhaseResults(
+                                            List<QuerySearchResult> querySearchResults,
+                                            SearchPhaseController.TopDocsStats topDocStats
+                                        ) {
+                                            List<TestRankDoc> rankDocs = new ArrayList<>();
+                                            for (int i = 0; i < querySearchResults.size(); i++) {
+                                                QuerySearchResult querySearchResult = querySearchResults.get(i);
+                                                TestRankShardResult shardResult = (TestRankShardResult) querySearchResult
+                                                    .getRankShardResult();
+                                                for (TestRankDoc trd : shardResult.testRankDocs) {
+                                                    trd.shardIndex = i;
+                                                    rankDocs.add(trd);
+                                                }
+                                            }
+                                            rankDocs.sort(Comparator.comparing((TestRankDoc doc) -> doc.score).reversed());
+                                            TestRankDoc[] topResults = rankDocs.stream().limit(rankWindowSize).toArray(TestRankDoc[]::new);
+                                            topDocStats.fetchHits = topResults.length;
+                                            return topResults;
+                                        }
+                                    };
+                                }
+
+                                @Override
+                                public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
+                                    return new QueryPhaseRankShardContext(queries, from) {
+
+                                        @Override
+                                        public int rankWindowSize() {
+                                            return DEFAULT_RANK_WINDOW_SIZE;
+                                        }
+
+                                        @Override
+                                        public RankShardResult combineQueryPhaseResults(List<TopDocs> rankResults) {
+                                            // we know we have just 1 query, so return all the docs from it
+                                            return new TestRankShardResult(
+                                                Arrays.stream(rankResults.get(0).scoreDocs)
+                                                    .map(x -> new TestRankDoc(x.doc, x.score, x.shardIndex))
+                                                    .limit(rankWindowSize())
+                                                    .toArray(TestRankDoc[]::new)
+                                            );
+                                        }
+                                    };
+                                }
+
+                                @Override
+                                public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+                                    return new RankFeaturePhaseRankShardContext(rankFeatureFieldName) {
+                                        @Override
+                                        public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) {
+                                            if (shardId == 0) {
+                                                throw new UnsupportedOperationException("simulated failure");
+                                            } else {
+                                                RankFeatureDoc[] rankFeatureDocs = new RankFeatureDoc[hits.getHits().length];
+                                                for (int i = 0; i < hits.getHits().length; i++) {
+                                                    SearchHit hit = hits.getHits()[i];
+                                                    rankFeatureDocs[i] = new RankFeatureDoc(hit.docId(), hit.getScore(), shardId);
+                                                    rankFeatureDocs[i].featureData(hit.getFields().get(rankFeatureFieldName).getValue());
+                                                    rankFeatureDocs[i].score = randomFloat();
+                                                    rankFeatureDocs[i].rank = i + 1;
+                                                }
+                                                return new RankFeatureShardResult(rankFeatureDocs);
+                                            }
+                                        }
+                                    };
+                                }
+                            }
+                        )
+                ),
+            (searchResponse) -> {
+                assertEquals(1, searchResponse.getSuccessfulShards());
+                assertEquals("simulated failure", searchResponse.getShardFailures()[0].getCause().getMessage());
+                assertNotEquals(0, searchResponse.getHits().getHits().length);
+                for (SearchHit hit : searchResponse.getHits().getHits()) {
+                    assertEquals(fetchFieldValue + "_" + hit.getId(), hit.getFields().get(fetchFieldName).getValue());
+                    assertEquals(1, hit.getShard().getShardId().id());
+                }
+            }
+        );
+    }
+
     public void testSearchWhileIndexDeletedDoesNotLeakSearchContext() throws ExecutionException, InterruptedException {
         createIndex("index");
         prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
@@ -457,7 +1182,7 @@ public void testSearchWhileIndexDeletedDoesNotLeakSearchContext() throws Executi
                 -1,
                 null
             ),
-            new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()),
+            new SearchShardTask(123L, "", "", "", null, emptyMap()),
             result
         );
 
@@ -694,7 +1419,7 @@ public void testMaxScriptFieldsSearch() throws IOException {
         for (int i = 0; i < maxScriptFields; i++) {
             searchSourceBuilder.scriptField(
                 "field" + i,
-                new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap())
+                new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, emptyMap())
             );
         }
         final ShardSearchRequest request = new ShardSearchRequest(
@@ -723,7 +1448,7 @@ public void testMaxScriptFieldsSearch() throws IOException {
             }
             searchSourceBuilder.scriptField(
                 "anotherScriptField",
-                new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap())
+                new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, emptyMap())
             );
             IllegalArgumentException ex = expectThrows(
                 IllegalArgumentException.class,
@@ -752,7 +1477,7 @@ public void testIgnoreScriptfieldIfSizeZero() throws IOException {
         searchRequest.source(searchSourceBuilder);
         searchSourceBuilder.scriptField(
             "field" + 0,
-            new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap())
+            new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, emptyMap())
         );
         searchSourceBuilder.size(0);
         final ShardSearchRequest request = new ShardSearchRequest(
@@ -1036,7 +1761,7 @@ public void testCanMatch() throws Exception {
         );
 
         CountDownLatch latch = new CountDownLatch(1);
-        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
+        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, emptyMap());
         // Because the foo field used in alias filter is unmapped the term query builder rewrite can resolve to a match no docs query,
         // without acquiring a searcher and that means the wrapper is not called
         assertEquals(5, numWrapInvocations.get());
@@ -1330,7 +2055,7 @@ public void testMatchNoDocsEmptyResponse() throws InterruptedException {
             0,
             null
         );
-        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
+        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, emptyMap());
 
         {
             CountDownLatch latch = new CountDownLatch(1);
@@ -1705,7 +2430,7 @@ public void testWaitOnRefresh() throws ExecutionException, InterruptedException
         final DocWriteResponse response = prepareIndex("index").setSource("id", "1").get();
         assertEquals(RestStatus.CREATED, response.status());
 
-        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
+        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, emptyMap());
         ShardSearchRequest request = new ShardSearchRequest(
             OriginalIndices.NONE,
             searchRequest,
@@ -1740,7 +2465,7 @@ public void testWaitOnRefreshFailsWithRefreshesDisabled() {
         final DocWriteResponse response = prepareIndex("index").setSource("id", "1").get();
         assertEquals(RestStatus.CREATED, response.status());
 
-        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
+        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, emptyMap());
         PlainActionFuture<SearchPhaseResult> future = new PlainActionFuture<>();
         ShardSearchRequest request = new ShardSearchRequest(
             OriginalIndices.NONE,
@@ -1778,7 +2503,7 @@ public void testWaitOnRefreshFailsIfCheckpointNotIndexed() {
         final DocWriteResponse response = prepareIndex("index").setSource("id", "1").get();
         assertEquals(RestStatus.CREATED, response.status());
 
-        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
+        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, emptyMap());
         PlainActionFuture<SearchPhaseResult> future = new PlainActionFuture<>();
         ShardSearchRequest request = new ShardSearchRequest(
             OriginalIndices.NONE,
@@ -1815,7 +2540,7 @@ public void testWaitOnRefreshTimeout() {
         final DocWriteResponse response = prepareIndex("index").setSource("id", "1").get();
         assertEquals(RestStatus.CREATED, response.status());
 
-        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, Collections.emptyMap());
+        SearchShardTask task = new SearchShardTask(123L, "", "", "", null, emptyMap());
         PlainActionFuture<SearchPhaseResult> future = new PlainActionFuture<>();
         ShardSearchRequest request = new ShardSearchRequest(
             OriginalIndices.NONE,
@@ -1901,7 +2626,7 @@ public void testDfsQueryPhaseRewrite() {
         PlainActionFuture<QuerySearchResult> plainActionFuture = new PlainActionFuture<>();
         service.executeQueryPhase(
             new QuerySearchRequest(null, context.id(), request, new AggregatedDfs(Map.of(), Map.of(), 10)),
-            new SearchShardTask(42L, "", "", "", null, Collections.emptyMap()),
+            new SearchShardTask(42L, "", "", "", null, emptyMap()),
             plainActionFuture
         );
 
diff --git a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
new file mode 100644
index 0000000000000..cf464044cd701
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
@@ -0,0 +1,409 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
+import org.elasticsearch.search.fetch.subphase.FetchFieldsContext;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
+import org.elasticsearch.search.rank.feature.RankFeatureDoc;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
+import org.elasticsearch.search.rank.feature.RankFeatureShardPhase;
+import org.elasticsearch.search.rank.feature.RankFeatureShardRequest;
+import org.elasticsearch.search.rank.feature.RankFeatureShardResult;
+import org.elasticsearch.tasks.TaskCancelledException;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestSearchContext;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+public class RankFeatureShardPhaseTests extends ESTestCase {
+
+    private SearchContext getSearchContext() {
+        return new TestSearchContext((SearchExecutionContext) null) {
+
+            private FetchSearchResult fetchResult;
+            private RankFeatureResult rankFeatureResult;
+            private FetchFieldsContext fetchFieldsContext;
+            private StoredFieldsContext storedFieldsContext;
+
+            @Override
+            public FetchSearchResult fetchResult() {
+                return fetchResult;
+            }
+
+            @Override
+            public void addFetchResult() {
+                this.fetchResult = new FetchSearchResult();
+                this.addReleasable(fetchResult::decRef);
+            }
+
+            @Override
+            public RankFeatureResult rankFeatureResult() {
+                return rankFeatureResult;
+            }
+
+            @Override
+            public void addRankFeatureResult() {
+                this.rankFeatureResult = new RankFeatureResult();
+                this.addReleasable(rankFeatureResult::decRef);
+            }
+
+            @Override
+            public SearchContext fetchFieldsContext(FetchFieldsContext fetchFieldsContext) {
+                this.fetchFieldsContext = fetchFieldsContext;
+                return this;
+            }
+
+            @Override
+            public FetchFieldsContext fetchFieldsContext() {
+                return fetchFieldsContext;
+            }
+
+            @Override
+            public SearchContext storedFieldsContext(StoredFieldsContext storedFieldsContext) {
+                this.storedFieldsContext = storedFieldsContext;
+                return this;
+            }
+
+            @Override
+            public StoredFieldsContext storedFieldsContext() {
+                return storedFieldsContext;
+            }
+
+            @Override
+            public boolean isCancelled() {
+                return false;
+            }
+        };
+    }
+
+    private RankBuilder getRankBuilder(final String field) {
+        return new RankBuilder(DEFAULT_RANK_WINDOW_SIZE) {
+            @Override
+            protected void doWriteTo(StreamOutput out) throws IOException {
+                // no-op
+            }
+
+            @Override
+            protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+                // no-op
+            }
+
+            @Override
+            public boolean isCompoundBuilder() {
+                return false;
+            }
+
+            // no work to be done on the query phase
+            @Override
+            public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
+                return null;
+            }
+
+            // no work to be done on the query phase
+            @Override
+            public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int size, int from) {
+                return null;
+            }
+
+            @Override
+            public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+                return new RankFeaturePhaseRankShardContext(field) {
+                    @Override
+                    public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) {
+                        RankFeatureDoc[] rankFeatureDocs = new RankFeatureDoc[hits.getHits().length];
+                        for (int i = 0; i < hits.getHits().length; i++) {
+                            SearchHit hit = hits.getHits()[i];
+                            rankFeatureDocs[i] = new RankFeatureDoc(hit.docId(), hit.getScore(), shardId);
+                            rankFeatureDocs[i].featureData(hit.getFields().get(field).getValue());
+                            rankFeatureDocs[i].rank = i + 1;
+                        }
+                        return new RankFeatureShardResult(rankFeatureDocs);
+                    }
+                };
+            }
+
+            // no work to be done on the coordinator node for the rank feature phase
+            @Override
+            public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from) {
+                return null;
+            }
+
+            @Override
+            protected boolean doEquals(RankBuilder other) {
+                return false;
+            }
+
+            @Override
+            protected int doHashCode() {
+                return 0;
+            }
+
+            @Override
+            public String getWriteableName() {
+                return "rank_builder_rank_feature_shard_phase_enabled";
+            }
+
+            @Override
+            public TransportVersion getMinimalSupportedVersion() {
+                return TransportVersions.RANK_FEATURE_PHASE_ADDED;
+            }
+        };
+    }
+
+    public void testPrepareForFetch() {
+
+        final String fieldName = "some_field";
+        int numDocs = randomIntBetween(10, 30);
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.rankBuilder(getRankBuilder(fieldName));
+
+        ShardSearchRequest searchRequest = mock(ShardSearchRequest.class);
+        when(searchRequest.source()).thenReturn(searchSourceBuilder);
+
+        try (SearchContext searchContext = spy(getSearchContext())) {
+            when(searchContext.isCancelled()).thenReturn(false);
+            when(searchContext.request()).thenReturn(searchRequest);
+
+            RankFeatureShardRequest request = mock(RankFeatureShardRequest.class);
+            when(request.getDocIds()).thenReturn(new int[] { 4, 9, numDocs - 1 });
+
+            RankFeatureShardPhase rankFeatureShardPhase = new RankFeatureShardPhase();
+            rankFeatureShardPhase.prepareForFetch(searchContext, request);
+
+            assertNotNull(searchContext.fetchFieldsContext());
+            assertEquals(searchContext.fetchFieldsContext().fields().size(), 1);
+            assertEquals(searchContext.fetchFieldsContext().fields().get(0).field, fieldName);
+            assertNotNull(searchContext.storedFieldsContext());
+            assertNull(searchContext.storedFieldsContext().fieldNames());
+            assertFalse(searchContext.storedFieldsContext().fetchFields());
+            assertNotNull(searchContext.fetchResult());
+        }
+    }
+
+    public void testPrepareForFetchNoRankFeatureContext() {
+        int numDocs = randomIntBetween(10, 30);
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.rankBuilder(null);
+
+        ShardSearchRequest searchRequest = mock(ShardSearchRequest.class);
+        when(searchRequest.source()).thenReturn(searchSourceBuilder);
+
+        try (SearchContext searchContext = spy(getSearchContext())) {
+            when(searchContext.isCancelled()).thenReturn(false);
+            when(searchContext.request()).thenReturn(searchRequest);
+
+            RankFeatureShardRequest request = mock(RankFeatureShardRequest.class);
+            when(request.getDocIds()).thenReturn(new int[] { 4, 9, numDocs - 1 });
+
+            RankFeatureShardPhase rankFeatureShardPhase = new RankFeatureShardPhase();
+            rankFeatureShardPhase.prepareForFetch(searchContext, request);
+
+            assertNull(searchContext.fetchFieldsContext());
+            assertNull(searchContext.fetchResult());
+        }
+    }
+
+    public void testPrepareForFetchWhileTaskIsCancelled() {
+
+        final String fieldName = "some_field";
+        int numDocs = randomIntBetween(10, 30);
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.rankBuilder(getRankBuilder(fieldName));
+
+        ShardSearchRequest searchRequest = mock(ShardSearchRequest.class);
+        when(searchRequest.source()).thenReturn(searchSourceBuilder);
+
+        try (SearchContext searchContext = spy(getSearchContext())) {
+            when(searchContext.isCancelled()).thenReturn(true);
+            when(searchContext.request()).thenReturn(searchRequest);
+
+            RankFeatureShardRequest request = mock(RankFeatureShardRequest.class);
+            when(request.getDocIds()).thenReturn(new int[] { 4, 9, numDocs - 1 });
+
+            RankFeatureShardPhase rankFeatureShardPhase = new RankFeatureShardPhase();
+            expectThrows(TaskCancelledException.class, () -> rankFeatureShardPhase.prepareForFetch(searchContext, request));
+        }
+    }
+
+    public void testProcessFetch() {
+        final String fieldName = "some_field";
+        int numDocs = randomIntBetween(10, 30);
+        Map<Integer, String> expectedFieldData = Map.of(4, "doc_4_aardvark", 9, "doc_9_aardvark", numDocs - 1, "last_doc_aardvark");
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.rankBuilder(getRankBuilder(fieldName));
+
+        ShardSearchRequest searchRequest = mock(ShardSearchRequest.class);
+        when(searchRequest.source()).thenReturn(searchSourceBuilder);
+
+        SearchShardTarget shardTarget = new SearchShardTarget(
+            "node_id",
+            new ShardId(new Index("some_index", UUID.randomUUID().toString()), 0),
+            null
+        );
+        SearchHits searchHits = null;
+        try (SearchContext searchContext = spy(getSearchContext())) {
+            searchContext.addFetchResult();
+            SearchHit[] hits = new SearchHit[3];
+            hits[0] = SearchHit.unpooled(4);
+            hits[0].setDocumentField(fieldName, new DocumentField(fieldName, Collections.singletonList(expectedFieldData.get(4))));
+
+            hits[1] = SearchHit.unpooled(9);
+            hits[1].setDocumentField(fieldName, new DocumentField(fieldName, Collections.singletonList(expectedFieldData.get(9))));
+
+            hits[2] = SearchHit.unpooled(numDocs - 1);
+            hits[2].setDocumentField(
+                fieldName,
+                new DocumentField(fieldName, Collections.singletonList(expectedFieldData.get(numDocs - 1)))
+            );
+            searchHits = SearchHits.unpooled(hits, new TotalHits(3, TotalHits.Relation.EQUAL_TO), 1.0f);
+            searchContext.fetchResult().shardResult(searchHits, null);
+            when(searchContext.isCancelled()).thenReturn(false);
+            when(searchContext.request()).thenReturn(searchRequest);
+            when(searchContext.shardTarget()).thenReturn(shardTarget);
+            RankFeatureShardRequest request = mock(RankFeatureShardRequest.class);
+            when(request.getDocIds()).thenReturn(new int[] { 4, 9, numDocs - 1 });
+
+            RankFeatureShardPhase rankFeatureShardPhase = new RankFeatureShardPhase();
+            // this is called as part of the search context initialization
+            // with the ResultsType.RANK_FEATURE type
+            searchContext.addRankFeatureResult();
+            rankFeatureShardPhase.processFetch(searchContext);
+
+            assertNotNull(searchContext.rankFeatureResult());
+            assertNotNull(searchContext.rankFeatureResult().rankFeatureResult());
+            for (RankFeatureDoc rankFeatureDoc : searchContext.rankFeatureResult().rankFeatureResult().shardResult().rankFeatureDocs) {
+                assertTrue(expectedFieldData.containsKey(rankFeatureDoc.doc));
+                assertEquals(rankFeatureDoc.featureData, expectedFieldData.get(rankFeatureDoc.doc));
+            }
+        } finally {
+            if (searchHits != null) {
+                searchHits.decRef();
+            }
+        }
+    }
+
+    public void testProcessFetchEmptyHits() {
+        final String fieldName = "some_field";
+        int numDocs = randomIntBetween(10, 30);
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.rankBuilder(getRankBuilder(fieldName));
+
+        ShardSearchRequest searchRequest = mock(ShardSearchRequest.class);
+        when(searchRequest.source()).thenReturn(searchSourceBuilder);
+
+        SearchShardTarget shardTarget = new SearchShardTarget(
+            "node_id",
+            new ShardId(new Index("some_index", UUID.randomUUID().toString()), 0),
+            null
+        );
+
+        SearchHits searchHits = null;
+        try (SearchContext searchContext = spy(getSearchContext())) {
+            searchContext.addFetchResult();
+            SearchHit[] hits = new SearchHit[0];
+            searchHits = SearchHits.unpooled(hits, new TotalHits(0, TotalHits.Relation.EQUAL_TO), 1.0f);
+            searchContext.fetchResult().shardResult(searchHits, null);
+            when(searchContext.isCancelled()).thenReturn(false);
+            when(searchContext.request()).thenReturn(searchRequest);
+            when(searchContext.shardTarget()).thenReturn(shardTarget);
+            RankFeatureShardRequest request = mock(RankFeatureShardRequest.class);
+            when(request.getDocIds()).thenReturn(new int[] { 4, 9, numDocs - 1 });
+
+            RankFeatureShardPhase rankFeatureShardPhase = new RankFeatureShardPhase();
+            // this is called as part of the search context initialization
+            // with the ResultsType.RANK_FEATURE type
+            searchContext.addRankFeatureResult();
+            rankFeatureShardPhase.processFetch(searchContext);
+
+            assertNotNull(searchContext.rankFeatureResult());
+            assertNotNull(searchContext.rankFeatureResult().rankFeatureResult());
+            assertEquals(searchContext.rankFeatureResult().rankFeatureResult().shardResult().rankFeatureDocs.length, 0);
+        } finally {
+            if (searchHits != null) {
+                searchHits.decRef();
+            }
+        }
+    }
+
+    public void testProcessFetchWhileTaskIsCancelled() {
+
+        final String fieldName = "some_field";
+        int numDocs = randomIntBetween(10, 30);
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.rankBuilder(getRankBuilder(fieldName));
+
+        ShardSearchRequest searchRequest = mock(ShardSearchRequest.class);
+        when(searchRequest.source()).thenReturn(searchSourceBuilder);
+
+        SearchShardTarget shardTarget = new SearchShardTarget(
+            "node_id",
+            new ShardId(new Index("some_index", UUID.randomUUID().toString()), 0),
+            null
+        );
+
+        SearchHits searchHits = null;
+        try (SearchContext searchContext = spy(getSearchContext())) {
+            searchContext.addFetchResult();
+            SearchHit[] hits = new SearchHit[0];
+            searchHits = SearchHits.unpooled(hits, new TotalHits(0, TotalHits.Relation.EQUAL_TO), 1.0f);
+            searchContext.fetchResult().shardResult(searchHits, null);
+            when(searchContext.isCancelled()).thenReturn(true);
+            when(searchContext.request()).thenReturn(searchRequest);
+            when(searchContext.shardTarget()).thenReturn(shardTarget);
+            RankFeatureShardRequest request = mock(RankFeatureShardRequest.class);
+            when(request.getDocIds()).thenReturn(new int[] { 4, 9, numDocs - 1 });
+
+            RankFeatureShardPhase rankFeatureShardPhase = new RankFeatureShardPhase();
+            // this is called as part of the search context initialization
+            // with the ResultsType.RANK_FEATURE type
+            searchContext.addRankFeatureResult();
+            expectThrows(TaskCancelledException.class, () -> rankFeatureShardPhase.processFetch(searchContext));
+        } finally {
+            if (searchHits != null) {
+                searchHits.decRef();
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index 697b40671ee8b..6419759ab5962 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -178,6 +178,7 @@
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.FetchPhase;
+import org.elasticsearch.search.rank.feature.RankFeatureShardPhase;
 import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.test.ClusterServiceUtils;
@@ -2249,6 +2250,7 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() {
                     threadPool,
                     scriptService,
                     bigArrays,
+                    new RankFeatureShardPhase(),
                     new FetchPhase(Collections.emptyList()),
                     responseCollectorService,
                     new NoneCircuitBreakerService(),
diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
index ef29f9fca4f93..520aff77497ba 100644
--- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.fetch.FetchPhase;
+import org.elasticsearch.search.rank.feature.RankFeatureShardPhase;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.test.ESTestCase;
@@ -97,6 +98,7 @@ SearchService newSearchService(
             ThreadPool threadPool,
             ScriptService scriptService,
             BigArrays bigArrays,
+            RankFeatureShardPhase rankFeatureShardPhase,
             FetchPhase fetchPhase,
             ResponseCollectorService responseCollectorService,
             CircuitBreakerService circuitBreakerService,
@@ -111,6 +113,7 @@ SearchService newSearchService(
                     threadPool,
                     scriptService,
                     bigArrays,
+                    rankFeatureShardPhase,
                     fetchPhase,
                     responseCollectorService,
                     circuitBreakerService,
@@ -124,6 +127,7 @@ SearchService newSearchService(
                 threadPool,
                 scriptService,
                 bigArrays,
+                rankFeatureShardPhase,
                 fetchPhase,
                 responseCollectorService,
                 circuitBreakerService,
diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
index aa1889e15d594..747eff1d21708 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.search.internal.ReaderContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.search.rank.feature.RankFeatureShardPhase;
 import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -81,6 +82,7 @@ public MockSearchService(
         ThreadPool threadPool,
         ScriptService scriptService,
         BigArrays bigArrays,
+        RankFeatureShardPhase rankFeatureShardPhase,
         FetchPhase fetchPhase,
         ResponseCollectorService responseCollectorService,
         CircuitBreakerService circuitBreakerService,
@@ -93,6 +95,7 @@ public MockSearchService(
             threadPool,
             scriptService,
             bigArrays,
+            rankFeatureShardPhase,
             fetchPhase,
             responseCollectorService,
             circuitBreakerService,
diff --git a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java
index 8e2a2c96a31ab..862c4d2ea3270 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java
@@ -15,6 +15,8 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -31,7 +33,7 @@ public class TestRankBuilder extends RankBuilder {
 
     static final ConstructingObjectParser<TestRankBuilder, Void> PARSER = new ConstructingObjectParser<>(
         NAME,
-        args -> new TestRankBuilder(args[0] == null ? DEFAULT_WINDOW_SIZE : (int) args[0])
+        args -> new TestRankBuilder(args[0] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[0])
     );
 
     static {
@@ -74,6 +76,11 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         // do nothing
     }
 
+    @Override
+    public boolean isCompoundBuilder() {
+        return true;
+    }
+
     @Override
     public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
         throw new UnsupportedOperationException();
@@ -84,6 +91,16 @@ public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int si
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from) {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     protected boolean doEquals(RankBuilder other) {
         return true;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index cba2b41d279bb..fa414cd8121d6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.feature.RankFeatureResult;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
@@ -463,6 +464,16 @@ public float getMaxScore() {
         return queryResult.getMaxScore();
     }
 
+    @Override
+    public void addRankFeatureResult() {
+        // this space intentionally left blank
+    }
+
+    @Override
+    public RankFeatureResult rankFeatureResult() {
+        return null;
+    }
+
     @Override
     public FetchSearchResult fetchResult() {
         return null;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index a7f21bd206c62..bf9eba87ee809 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -687,6 +687,10 @@ public static Matcher<SearchHit> hasScore(final float score) {
         return transformedMatch(SearchHit::getScore, equalTo(score));
     }
 
+    public static Matcher<SearchHit> hasRank(final int rank) {
+        return transformedMatch(SearchHit::getRank, equalTo(rank));
+    }
+
     public static <T extends Query> T assertBooleanSubQuery(Query query, Class<T> subqueryType, int i) {
         assertThat(query, instanceOf(BooleanQuery.class));
         BooleanQuery q = (BooleanQuery) query;
diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
index 04b0b11ad38d4..c0305f873327d 100644
--- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
+++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java
@@ -397,6 +397,14 @@ protected void onQueryResult(int shardIndex, QuerySearchResult queryResult) {
             }
         }
 
+        @Override
+        protected void onRankFeatureResult(int shardIndex) {
+            checkCancellation();
+            if (delegate != null) {
+                delegate.onRankFeatureResult(shardIndex);
+            }
+        }
+
         @Override
         protected void onFetchResult(int shardIndex) {
             checkCancellation();
@@ -420,6 +428,12 @@ protected void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exc
                 );
         }
 
+        @Override
+        protected void onRankFeatureFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
+            // best effort to cancel expired tasks
+            checkCancellation();
+        }
+
         @Override
         protected void onFetchFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {
             // best effort to cancel expired tasks
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
index 8f3ed15037c08..5c39c6c32fd06 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
@@ -16,6 +16,8 @@
 import org.elasticsearch.search.rank.RankBuilder;
 import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
+import org.elasticsearch.search.rank.context.RankFeaturePhaseRankShardContext;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -38,7 +40,7 @@ public class RRFRankBuilder extends RankBuilder {
     public static final ParseField RANK_CONSTANT_FIELD = new ParseField("rank_constant");
 
     static final ConstructingObjectParser<RRFRankBuilder, Void> PARSER = new ConstructingObjectParser<>(RRFRankPlugin.NAME, args -> {
-        int windowSize = args[0] == null ? DEFAULT_WINDOW_SIZE : (int) args[0];
+        int windowSize = args[0] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[0];
         int rankConstant = args[1] == null ? DEFAULT_RANK_CONSTANT : (int) args[1];
         if (rankConstant < 1) {
             throw new IllegalArgumentException("[rank_constant] must be greater than [0] for [rrf]");
@@ -94,6 +96,11 @@ public int rankConstant() {
         return rankConstant;
     }
 
+    @Override
+    public boolean isCompoundBuilder() {
+        return true;
+    }
+
     public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
         return new RRFQueryPhaseRankShardContext(queries, rankWindowSize(), rankConstant);
     }
@@ -103,6 +110,16 @@ public QueryPhaseRankCoordinatorContext buildQueryPhaseCoordinatorContext(int si
         return new RRFQueryPhaseRankCoordinatorContext(size, from, rankWindowSize(), rankConstant);
     }
 
+    @Override
+    public RankFeaturePhaseRankShardContext buildRankFeaturePhaseShardContext() {
+        return null;
+    }
+
+    @Override
+    public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from) {
+        return null;
+    }
+
     @Override
     protected boolean doEquals(RankBuilder other) {
         return Objects.equals(rankConstant, ((RRFRankBuilder) other).rankConstant);
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
index 077c933fa9add..e5a7983107278 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java
@@ -71,7 +71,7 @@ public static RRFRetrieverBuilder fromXContent(XContentParser parser, RetrieverP
     }
 
     List<RetrieverBuilder> retrieverBuilders = Collections.emptyList();
-    int rankWindowSize = RRFRankBuilder.DEFAULT_WINDOW_SIZE;
+    int rankWindowSize = RRFRankBuilder.DEFAULT_RANK_WINDOW_SIZE;
     int rankConstant = RRFRankBuilder.DEFAULT_RANK_CONSTANT;
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/PreAuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/PreAuthorizationUtils.java
index aeb6bfc8de796..221b7a65e1f8f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/PreAuthorizationUtils.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/PreAuthorizationUtils.java
@@ -45,6 +45,7 @@ public final class PreAuthorizationUtils {
             SearchTransportService.QUERY_ACTION_NAME,
             SearchTransportService.QUERY_ID_ACTION_NAME,
             SearchTransportService.FETCH_ID_ACTION_NAME,
+            SearchTransportService.RANK_FEATURE_SHARD_ACTION_NAME,
             SearchTransportService.QUERY_CAN_MATCH_NODE_NAME
         )
     );

From ed0febb8d56e81edbd5214d50503e6880fb6f9f8 Mon Sep 17 00:00:00 2001
From: Albert Zaharovits <albert.zaharovits@elastic.co>
Date: Thu, 6 Jun 2024 14:53:04 +0300
Subject: [PATCH 190/208] Fix task cancellation authz on fulfilling cluster
 (#109357)

This fixes task cancellation actions (i.e. internal:admin/tasks/cancel_child
and internal:admin/tasks/ban) not being authorized by the fulfilling cluster.
This can result in orphaned tasks on the fulfilling cluster.
---
 docs/changelog/109357.yaml                    |   5 +
 .../tasks/TaskCancellationService.java        |  14 ++
 .../privilege/ClusterPrivilegeResolver.java   |   5 +
 .../RemoteClusterSecurityRestIT.java          | 190 ++++++++++++++++++
 .../security/authz/AuthorizationService.java  |   2 +-
 .../SecurityServerTransportInterceptor.java   |   7 +-
 6 files changed, 221 insertions(+), 2 deletions(-)
 create mode 100644 docs/changelog/109357.yaml

diff --git a/docs/changelog/109357.yaml b/docs/changelog/109357.yaml
new file mode 100644
index 0000000000000..17951882103b3
--- /dev/null
+++ b/docs/changelog/109357.yaml
@@ -0,0 +1,5 @@
+pr: 109357
+summary: Fix task cancellation authz on fulfilling cluster
+area: Authorization
+type: bug
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java
index 419f2d0726880..d292a6a15a79a 100644
--- a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java
+++ b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java
@@ -47,7 +47,9 @@
 
 public class TaskCancellationService {
     public static final String BAN_PARENT_ACTION_NAME = "internal:admin/tasks/ban";
+    public static final String REMOTE_CLUSTER_BAN_PARENT_ACTION_NAME = "cluster:internal/admin/tasks/ban";
     public static final String CANCEL_CHILD_ACTION_NAME = "internal:admin/tasks/cancel_child";
+    public static final String REMOTE_CLUSTER_CANCEL_CHILD_ACTION_NAME = "cluster:internal/admin/tasks/cancel_child";
     public static final TransportVersion VERSION_SUPPORTING_CANCEL_CHILD_ACTION = TransportVersions.V_8_8_0;
     private static final Logger logger = LogManager.getLogger(TaskCancellationService.class);
     private final TransportService transportService;
@@ -64,12 +66,24 @@ public TaskCancellationService(TransportService transportService) {
             BanParentTaskRequest::new,
             new BanParentRequestHandler()
         );
+        transportService.registerRequestHandler(
+            REMOTE_CLUSTER_BAN_PARENT_ACTION_NAME,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE,
+            BanParentTaskRequest::new,
+            new BanParentRequestHandler()
+        );
         transportService.registerRequestHandler(
             CANCEL_CHILD_ACTION_NAME,
             EsExecutors.DIRECT_EXECUTOR_SERVICE,
             CancelChildRequest::new,
             new CancelChildRequestHandler()
         );
+        transportService.registerRequestHandler(
+            REMOTE_CLUSTER_CANCEL_CHILD_ACTION_NAME,
+            EsExecutors.DIRECT_EXECUTOR_SERVICE,
+            CancelChildRequest::new,
+            new CancelChildRequestHandler()
+        );
     }
 
     private String localNodeId() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
index 7f927d45a2375..5b3829a00468b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.tasks.TaskCancellationService;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.action.XPackInfoAction;
@@ -183,6 +184,8 @@ public class ClusterPrivilegeResolver {
     private static final Set<String> CROSS_CLUSTER_SEARCH_PATTERN = Set.of(
         RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME,
         RemoteClusterNodesAction.TYPE.name(),
+        TaskCancellationService.REMOTE_CLUSTER_BAN_PARENT_ACTION_NAME,
+        TaskCancellationService.REMOTE_CLUSTER_CANCEL_CHILD_ACTION_NAME,
         XPackInfoAction.NAME,
         // esql enrich
         "cluster:monitor/xpack/enrich/esql/resolve_policy",
@@ -192,6 +195,8 @@ public class ClusterPrivilegeResolver {
     private static final Set<String> CROSS_CLUSTER_REPLICATION_PATTERN = Set.of(
         RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME,
         RemoteClusterNodesAction.TYPE.name(),
+        TaskCancellationService.REMOTE_CLUSTER_BAN_PARENT_ACTION_NAME,
+        TaskCancellationService.REMOTE_CLUSTER_CANCEL_CHILD_ACTION_NAME,
         XPackInfoAction.NAME,
         ClusterStateAction.NAME
     );
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestIT.java
index 6eb49ec1ab8ae..69331fa448113 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestIT.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.remotecluster;
 
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Build;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
@@ -15,13 +16,16 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchResponseUtils;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.junit.RunnableTestRuleAdapter;
 import org.elasticsearch.xcontent.ObjectPath;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
@@ -37,6 +41,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.anEmptyMap;
@@ -59,6 +64,7 @@ public class RemoteClusterSecurityRestIT extends AbstractRemoteClusterSecurityTe
 
     static {
         fulfillingCluster = ElasticsearchCluster.local()
+            .distribution(DistributionType.DEFAULT)
             .name("fulfilling-cluster")
             .nodes(3)
             .apply(commonClusterConfig)
@@ -74,6 +80,7 @@ public class RemoteClusterSecurityRestIT extends AbstractRemoteClusterSecurityTe
             .build();
 
         queryCluster = ElasticsearchCluster.local()
+            .distribution(DistributionType.DEFAULT)
             .name("query-cluster")
             .apply(commonClusterConfig)
             .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
@@ -138,6 +145,169 @@ public class RemoteClusterSecurityRestIT extends AbstractRemoteClusterSecurityTe
         INVALID_SECRET_LENGTH.set(randomValueOtherThan(22, () -> randomIntBetween(0, 99)));
     })).around(fulfillingCluster).around(queryCluster);
 
+    public void testTaskCancellation() throws Exception {
+        assumeTrue("[error_query] is only available in snapshot builds", Build.current().isSnapshot());
+        configureRemoteCluster();
+
+        final String indexName = "index_fulfilling";
+        final String roleName = "taskCancellationRoleName";
+        final String userName = "taskCancellationUsername";
+        try {
+            // create some index on the fulfilling cluster, to be searched from the querying cluster
+            {
+                Request bulkRequest = new Request("POST", "/_bulk?refresh=true");
+                bulkRequest.setJsonEntity(Strings.format("""
+                    { "index": { "_index": "%s" } }
+                    { "foo": "bar" }
+                    """, indexName));
+                assertOK(performRequestAgainstFulfillingCluster(bulkRequest));
+            }
+
+            // Create user and role with privileges for remote indices
+            var putRoleRequest = new Request("PUT", "/_security/role/" + roleName);
+            putRoleRequest.setJsonEntity(Strings.format("""
+                {
+                  "description": "Role with privileges for remote index for the test of task cancellation.",
+                  "remote_indices": [
+                    {
+                      "names": ["%s"],
+                      "privileges": ["read", "read_cross_cluster"],
+                      "clusters": ["my_remote_cluster"]
+                    }
+                  ]
+                }""", indexName));
+            assertOK(adminClient().performRequest(putRoleRequest));
+            var putUserRequest = new Request("PUT", "/_security/user/" + userName);
+            putUserRequest.setJsonEntity(Strings.format("""
+                {
+                  "password": "%s",
+                  "roles" : ["%s"]
+                }""", PASS, roleName));
+            assertOK(adminClient().performRequest(putUserRequest));
+            var submitAsyncSearchRequest = new Request(
+                "POST",
+                Strings.format(
+                    "/%s:%s/_async_search?ccs_minimize_roundtrips=%s",
+                    randomFrom("my_remote_cluster", "*", "my_remote_*"),
+                    indexName,
+                    randomBoolean()
+                )
+            );
+
+            // submit a stalling remote async search
+            submitAsyncSearchRequest.setJsonEntity("""
+                {
+                  "query": {
+                    "error_query": {
+                      "indices": [
+                        {
+                          "name": "*:*",
+                          "error_type": "exception",
+                          "stall_time_seconds": 60
+                        }
+                      ]
+                    }
+                  }
+                }""");
+            String asyncSearchOpaqueId = "async-search-opaque-id-" + randomUUID();
+            submitAsyncSearchRequest.setOptions(
+                RequestOptions.DEFAULT.toBuilder()
+                    .addHeader("Authorization", headerFromRandomAuthMethod(userName, PASS))
+                    .addHeader("X-Opaque-Id", asyncSearchOpaqueId)
+            );
+            Response submitAsyncSearchResponse = client().performRequest(submitAsyncSearchRequest);
+            assertOK(submitAsyncSearchResponse);
+            Map<String, Object> submitAsyncSearchResponseMap = XContentHelper.convertToMap(
+                JsonXContent.jsonXContent,
+                EntityUtils.toString(submitAsyncSearchResponse.getEntity()),
+                false
+            );
+            assertThat(submitAsyncSearchResponseMap.get("is_running"), equalTo(true));
+            String asyncSearchId = (String) submitAsyncSearchResponseMap.get("id");
+            assertThat(asyncSearchId, notNullValue());
+            // wait for the tasks to show up on the querying cluster
+            assertBusy(() -> {
+                try {
+                    Response queryingClusterTasks = adminClient().performRequest(new Request("GET", "/_tasks"));
+                    assertOK(queryingClusterTasks);
+                    Map<String, Object> responseMap = XContentHelper.convertToMap(
+                        JsonXContent.jsonXContent,
+                        EntityUtils.toString(queryingClusterTasks.getEntity()),
+                        false
+                    );
+                    AtomicBoolean someTasks = new AtomicBoolean(false);
+                    selectTasksWithOpaqueId(responseMap, asyncSearchOpaqueId, task -> {
+                        // search tasks should not be cancelled at this point (but some transitory ones might be,
+                        // e.g. for action "indices:admin/seq_no/global_checkpoint_sync")
+                        if (task.get("action") instanceof String action && action.contains("indices:data/read/search")) {
+                            assertThat(task.get("cancelled"), equalTo(false));
+                            someTasks.set(true);
+                        }
+                    });
+                    assertTrue(someTasks.get());
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+            // wait for the tasks to show up on the fulfilling cluster
+            assertBusy(() -> {
+                try {
+                    Response fulfillingClusterTasks = performRequestAgainstFulfillingCluster(new Request("GET", "/_tasks"));
+                    assertOK(fulfillingClusterTasks);
+                    Map<String, Object> responseMap = XContentHelper.convertToMap(
+                        JsonXContent.jsonXContent,
+                        EntityUtils.toString(fulfillingClusterTasks.getEntity()),
+                        false
+                    );
+                    AtomicBoolean someTasks = new AtomicBoolean(false);
+                    selectTasksWithOpaqueId(responseMap, asyncSearchOpaqueId, task -> {
+                        // search tasks should not be cancelled at this point (but some transitory ones might be,
+                        // e.g. for action "indices:admin/seq_no/global_checkpoint_sync")
+                        if (task.get("action") instanceof String action && action.contains("indices:data/read/search")) {
+                            assertThat(task.get("cancelled"), equalTo(false));
+                            someTasks.set(true);
+                        }
+                    });
+                    assertTrue(someTasks.get());
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+            // delete the stalling async search
+            var deleteAsyncSearchRequest = new Request("DELETE", Strings.format("/_async_search/%s", asyncSearchId));
+            deleteAsyncSearchRequest.setOptions(
+                RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", headerFromRandomAuthMethod(userName, PASS))
+            );
+            assertOK(client().performRequest(deleteAsyncSearchRequest));
+            // ensure any remaining tasks are all cancelled on the querying cluster
+            {
+                Response queryingClusterTasks = adminClient().performRequest(new Request("GET", "/_tasks"));
+                assertOK(queryingClusterTasks);
+                Map<String, Object> responseMap = XContentHelper.convertToMap(
+                    JsonXContent.jsonXContent,
+                    EntityUtils.toString(queryingClusterTasks.getEntity()),
+                    false
+                );
+                selectTasksWithOpaqueId(responseMap, asyncSearchOpaqueId, task -> assertThat(task.get("cancelled"), equalTo(true)));
+            }
+            // ensure any remaining tasks are all cancelled on the fulfilling cluster
+            {
+                Response fulfillingClusterTasks = performRequestAgainstFulfillingCluster(new Request("GET", "/_tasks"));
+                assertOK(fulfillingClusterTasks);
+                Map<String, Object> responseMap = XContentHelper.convertToMap(
+                    JsonXContent.jsonXContent,
+                    EntityUtils.toString(fulfillingClusterTasks.getEntity()),
+                    false
+                );
+                selectTasksWithOpaqueId(responseMap, asyncSearchOpaqueId, task -> assertThat(task.get("cancelled"), equalTo(true)));
+            }
+        } finally {
+            assertOK(adminClient().performRequest(new Request("DELETE", "/_security/user/" + userName)));
+            assertOK(adminClient().performRequest(new Request("DELETE", "/_security/role/" + roleName)));
+            assertOK(performRequestAgainstFulfillingCluster(new Request("DELETE", indexName)));
+        }
+    }
+
     public void testCrossClusterSearch() throws Exception {
         configureRemoteCluster();
         final String crossClusterAccessApiKeyId = (String) API_KEY_MAP_REF.get().get("id");
@@ -491,4 +661,24 @@ private Response performRequestWithLocalSearchUser(final Request request) throws
         );
         return client().performRequest(request);
     }
+
+    @SuppressWarnings("unchecked")
+    private static void selectTasksWithOpaqueId(
+        Map<String, Object> tasksResponse,
+        String opaqueId,
+        Consumer<Map<String, Object>> taskConsumer
+    ) {
+        Map<String, Map<String, Object>> nodes = (Map<String, Map<String, Object>>) tasksResponse.get("nodes");
+        for (Map<String, Object> node : nodes.values()) {
+            Map<String, Map<String, Object>> tasks = (Map<String, Map<String, Object>>) node.get("tasks");
+            for (Map<String, Object> task : tasks.values()) {
+                if (task.get("headers") != null) {
+                    Map<String, Object> headers = (Map<String, Object>) task.get("headers");
+                    if (opaqueId.equals(headers.get("X-Opaque-Id"))) {
+                        taskConsumer.accept(task);
+                    }
+                }
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
index 718602b758072..0dfdf0861e321 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
@@ -533,7 +533,7 @@ private void authorizeAction(
                 )
             );
         } else {
-            logger.warn("denying access as action [{}] is not an index or cluster action", action);
+            logger.warn("denying access for [{}] as action [{}] is not an index or cluster action", authentication, action);
             auditTrail.accessDenied(requestId, authentication, action, request, authzInfo);
             listener.onFailure(actionDenied(authentication, authzInfo, action, request));
         }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
index d8914d4646fd3..268f9e6375f0e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskCancellationService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteConnectionManager;
 import org.elasticsearch.transport.RemoteConnectionManager.RemoteClusterAliasWithCredentials;
@@ -81,7 +82,11 @@ public class SecurityServerTransportInterceptor implements TransportInterceptor
         "internal:data/read/esql/open_exchange",
         "cluster:internal:data/read/esql/open_exchange",
         "internal:data/read/esql/exchange",
-        "cluster:internal:data/read/esql/exchange"
+        "cluster:internal:data/read/esql/exchange",
+        TaskCancellationService.BAN_PARENT_ACTION_NAME,
+        TaskCancellationService.REMOTE_CLUSTER_BAN_PARENT_ACTION_NAME,
+        TaskCancellationService.CANCEL_CHILD_ACTION_NAME,
+        TaskCancellationService.REMOTE_CLUSTER_CANCEL_CHILD_ACTION_NAME
     );
 
     private final AuthenticationService authcService;

From 3d7107d836042613098ea50398381399e05409b8 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Thu, 6 Jun 2024 15:25:24 +0200
Subject: [PATCH 191/208] [Inference API] Inline XContentParserUtils imports in
 response entities (#109421)

---
 ...eAiStudioChatCompletionResponseEntity.java |  3 +--
 .../CohereCompletionResponseEntity.java       |  3 +--
 .../CohereEmbeddingsResponseEntity.java       | 21 ++++++++++---------
 .../cohere/CohereRankedResponseEntity.java    | 11 +++++-----
 ...oogleAiStudioEmbeddingsResponseEntity.java | 13 ++++++------
 .../HuggingFaceElserResponseEntity.java       | 11 +++++-----
 .../HuggingFaceEmbeddingsResponseEntity.java  | 19 +++++++----------
 .../OpenAiEmbeddingsResponseEntity.java       | 13 ++++++------
 8 files changed, 46 insertions(+), 48 deletions(-)

diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioChatCompletionResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioChatCompletionResponseEntity.java
index 18f5923353960..abf2c4877307c 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioChatCompletionResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioChatCompletionResponseEntity.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.inference.external.response.azureaistudio;
 
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
@@ -49,7 +48,7 @@ private ChatCompletionResults parseRealtimeEndpointResponse(HttpResult response)
             moveToFirstToken(jsonParser);
 
             XContentParser.Token token = jsonParser.currentToken();
-            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
 
             while (token != null && token != XContentParser.Token.END_OBJECT) {
                 if (token != XContentParser.Token.FIELD_NAME) {
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereCompletionResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereCompletionResponseEntity.java
index 4740c93ea6c03..af58274d22181 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereCompletionResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereCompletionResponseEntity.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.inference.external.response.cohere;
 
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
@@ -84,7 +83,7 @@ public static ChatCompletionResults fromResponse(Request request, HttpResult res
             moveToFirstToken(jsonParser);
 
             XContentParser.Token token = jsonParser.currentToken();
-            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
 
             positionParserAtTokenAfterField(jsonParser, "text", FAILED_TO_FIND_FIELD_TEMPLATE);
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java
index f787c6337d646..88fe54c338bd5 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java
@@ -11,7 +11,6 @@
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -29,6 +28,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
@@ -140,7 +141,7 @@ public static InferenceServiceResults fromResponse(Request request, HttpResult r
             moveToFirstToken(jsonParser);
 
             XContentParser.Token token = jsonParser.currentToken();
-            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
 
             positionParserAtTokenAfterField(jsonParser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE);
 
@@ -183,21 +184,21 @@ private static InferenceServiceResults parseEmbeddingsObject(XContentParser pars
     }
 
     private static InferenceServiceResults parseByteEmbeddingsArray(XContentParser parser) throws IOException {
-        var embeddingList = XContentParserUtils.parseList(parser, CohereEmbeddingsResponseEntity::parseByteArrayEntry);
+        var embeddingList = parseList(parser, CohereEmbeddingsResponseEntity::parseByteArrayEntry);
 
         return new TextEmbeddingByteResults(embeddingList);
     }
 
     private static TextEmbeddingByteResults.Embedding parseByteArrayEntry(XContentParser parser) throws IOException {
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
-        List<Byte> embeddingValuesList = XContentParserUtils.parseList(parser, CohereEmbeddingsResponseEntity::parseEmbeddingInt8Entry);
+        ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
+        List<Byte> embeddingValuesList = parseList(parser, CohereEmbeddingsResponseEntity::parseEmbeddingInt8Entry);
 
         return TextEmbeddingByteResults.Embedding.of(embeddingValuesList);
     }
 
     private static Byte parseEmbeddingInt8Entry(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
+        ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
         var parsedByte = parser.shortValue();
         checkByteBounds(parsedByte);
 
@@ -211,20 +212,20 @@ private static void checkByteBounds(short value) {
     }
 
     private static InferenceServiceResults parseFloatEmbeddingsArray(XContentParser parser) throws IOException {
-        var embeddingList = XContentParserUtils.parseList(parser, CohereEmbeddingsResponseEntity::parseFloatArrayEntry);
+        var embeddingList = parseList(parser, CohereEmbeddingsResponseEntity::parseFloatArrayEntry);
 
         return new TextEmbeddingResults(embeddingList);
     }
 
     private static TextEmbeddingResults.Embedding parseFloatArrayEntry(XContentParser parser) throws IOException {
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
-        List<Float> embeddingValuesList = XContentParserUtils.parseList(parser, CohereEmbeddingsResponseEntity::parseEmbeddingFloatEntry);
+        ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
+        List<Float> embeddingValuesList = parseList(parser, CohereEmbeddingsResponseEntity::parseEmbeddingFloatEntry);
         return TextEmbeddingResults.Embedding.of(embeddingValuesList);
     }
 
     private static Float parseEmbeddingFloatEntry(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
+        ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
         return parser.floatValue();
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereRankedResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereRankedResponseEntity.java
index c9cc71b7fdcda..7f71933676ee0 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereRankedResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereRankedResponseEntity.java
@@ -12,7 +12,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
@@ -23,7 +22,9 @@
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
@@ -91,7 +92,7 @@ public static InferenceServiceResults fromResponse(HttpResult response) throws I
             moveToFirstToken(jsonParser);
 
             XContentParser.Token token = jsonParser.currentToken();
-            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
 
             positionParserAtTokenAfterField(jsonParser, "results", FAILED_TO_FIND_FIELD_TEMPLATE); // TODO error message
 
@@ -109,7 +110,7 @@ public static InferenceServiceResults fromResponse(HttpResult response) throws I
     }
 
     private static RankedDocsResults.RankedDoc parseRankedDocObject(XContentParser parser) throws IOException {
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
         int index = -1;
         float relevanceScore = -1;
         String documentText = null;
@@ -129,7 +130,7 @@ private static RankedDocsResults.RankedDoc parseRankedDocObject(XContentParser p
                         break;
                     case "document":
                         parser.nextToken(); // move to START_OBJECT; document text is wrapped in an object
-                        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+                        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
                         do {
                             if (parser.currentToken() == XContentParser.Token.FIELD_NAME && parser.currentName().equals("text")) {
                                 parser.nextToken(); // move to VALUE_STRING
@@ -140,7 +141,7 @@ private static RankedDocsResults.RankedDoc parseRankedDocObject(XContentParser p
                         // parser should now be at the next FIELD_NAME or END_OBJECT
                         break;
                     default:
-                        XContentParserUtils.throwUnknownField(parser.currentName(), parser);
+                        throwUnknownField(parser.currentName(), parser);
                 }
             } else {
                 parser.nextToken();
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java
index 204738f2a2552..d214aea42fb06 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.inference.external.response.googleaistudio;
 
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
@@ -20,6 +19,8 @@
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.consumeUntilObjectEnd;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
@@ -75,11 +76,11 @@ public static TextEmbeddingResults fromResponse(Request request, HttpResult resp
             moveToFirstToken(jsonParser);
 
             XContentParser.Token token = jsonParser.currentToken();
-            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
 
             positionParserAtTokenAfterField(jsonParser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-            List<TextEmbeddingResults.Embedding> embeddingList = XContentParserUtils.parseList(
+            List<TextEmbeddingResults.Embedding> embeddingList = parseList(
                 jsonParser,
                 GoogleAiStudioEmbeddingsResponseEntity::parseEmbeddingObject
             );
@@ -89,11 +90,11 @@ public static TextEmbeddingResults fromResponse(Request request, HttpResult resp
     }
 
     private static TextEmbeddingResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException {
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
 
         positionParserAtTokenAfterField(parser, "values", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-        List<Float> embeddingValuesList = XContentParserUtils.parseList(parser, GoogleAiStudioEmbeddingsResponseEntity::parseEmbeddingList);
+        List<Float> embeddingValuesList = parseList(parser, GoogleAiStudioEmbeddingsResponseEntity::parseEmbeddingList);
         // parse and discard the rest of the object
         consumeUntilObjectEnd(parser);
 
@@ -102,7 +103,7 @@ private static TextEmbeddingResults.Embedding parseEmbeddingObject(XContentParse
 
     private static float parseEmbeddingList(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
+        ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
         return parser.floatValue();
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntity.java
index 270a981a6998d..f0e729e15b615 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntity.java
@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.inference.external.response.huggingface;
 
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
@@ -23,6 +22,8 @@
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
 
 public class HuggingFaceElserResponseEntity {
@@ -65,7 +66,7 @@ public static SparseEmbeddingResults fromResponse(Request request, HttpResult re
             moveToFirstToken(jsonParser);
 
             var truncationResults = request.getTruncationInfo();
-            List<SparseEmbeddingResults.Embedding> parsedEmbeddings = XContentParserUtils.parseList(
+            List<SparseEmbeddingResults.Embedding> parsedEmbeddings = parseList(
                 jsonParser,
                 (parser, index) -> HuggingFaceElserResponseEntity.parseExpansionResult(truncationResults, parser, index)
             );
@@ -81,14 +82,14 @@ public static SparseEmbeddingResults fromResponse(Request request, HttpResult re
     private static SparseEmbeddingResults.Embedding parseExpansionResult(boolean[] truncationResults, XContentParser parser, int index)
         throws IOException {
         XContentParser.Token token = parser.currentToken();
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
 
         List<WeightedToken> weightedTokens = new ArrayList<>();
         token = parser.nextToken();
         while (token != null && token != XContentParser.Token.END_OBJECT) {
-            XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
             var floatToken = parser.nextToken();
-            XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, floatToken, parser);
+            ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, floatToken, parser);
 
             weightedTokens.add(new WeightedToken(parser.currentName(), parser.floatValue()));
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java
index a3e06b3c2075a..98c4ec7667fd0 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.xpack.inference.external.response.huggingface;
 
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
@@ -22,6 +21,8 @@
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
@@ -92,10 +93,7 @@ public static TextEmbeddingResults fromResponse(Request request, HttpResult resp
      * <a href="https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2">sentence-transformers/all-MiniLM-L12-v2</a>
      */
     private static TextEmbeddingResults parseArrayFormat(XContentParser parser) throws IOException {
-        List<TextEmbeddingResults.Embedding> embeddingList = XContentParserUtils.parseList(
-            parser,
-            HuggingFaceEmbeddingsResponseEntity::parseEmbeddingEntry
-        );
+        List<TextEmbeddingResults.Embedding> embeddingList = parseList(parser, HuggingFaceEmbeddingsResponseEntity::parseEmbeddingEntry);
 
         return new TextEmbeddingResults(embeddingList);
     }
@@ -139,24 +137,21 @@ private static TextEmbeddingResults parseArrayFormat(XContentParser parser) thro
     private static TextEmbeddingResults parseObjectFormat(XContentParser parser) throws IOException {
         positionParserAtTokenAfterField(parser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-        List<TextEmbeddingResults.Embedding> embeddingList = XContentParserUtils.parseList(
-            parser,
-            HuggingFaceEmbeddingsResponseEntity::parseEmbeddingEntry
-        );
+        List<TextEmbeddingResults.Embedding> embeddingList = parseList(parser, HuggingFaceEmbeddingsResponseEntity::parseEmbeddingEntry);
 
         return new TextEmbeddingResults(embeddingList);
     }
 
     private static TextEmbeddingResults.Embedding parseEmbeddingEntry(XContentParser parser) throws IOException {
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
+        ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
 
-        List<Float> embeddingValuesList = XContentParserUtils.parseList(parser, HuggingFaceEmbeddingsResponseEntity::parseEmbeddingList);
+        List<Float> embeddingValuesList = parseList(parser, HuggingFaceEmbeddingsResponseEntity::parseEmbeddingList);
         return TextEmbeddingResults.Embedding.of(embeddingValuesList);
     }
 
     private static float parseEmbeddingList(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
+        ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
         return parser.floatValue();
     }
 
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java
index 39b97014c3619..711944ea820a9 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.xpack.inference.external.response.openai;
 
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
@@ -22,6 +21,8 @@
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.consumeUntilObjectEnd;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken;
 import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField;
@@ -79,11 +80,11 @@ public static TextEmbeddingResults fromResponse(Request request, HttpResult resp
             moveToFirstToken(jsonParser);
 
             XContentParser.Token token = jsonParser.currentToken();
-            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser);
 
             positionParserAtTokenAfterField(jsonParser, "data", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-            List<TextEmbeddingResults.Embedding> embeddingList = XContentParserUtils.parseList(
+            List<TextEmbeddingResults.Embedding> embeddingList = parseList(
                 jsonParser,
                 OpenAiEmbeddingsResponseEntity::parseEmbeddingObject
             );
@@ -93,11 +94,11 @@ public static TextEmbeddingResults fromResponse(Request request, HttpResult resp
     }
 
     private static TextEmbeddingResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException {
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
 
         positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-        List<Float> embeddingValuesList = XContentParserUtils.parseList(parser, OpenAiEmbeddingsResponseEntity::parseEmbeddingList);
+        List<Float> embeddingValuesList = parseList(parser, OpenAiEmbeddingsResponseEntity::parseEmbeddingList);
         // parse and discard the rest of the object
         consumeUntilObjectEnd(parser);
 
@@ -106,7 +107,7 @@ private static TextEmbeddingResults.Embedding parseEmbeddingObject(XContentParse
 
     private static float parseEmbeddingList(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
-        XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
+        ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
         return parser.floatValue();
     }
 

From 283d78aa84e7a1b3960ae25dc8f4be56fe4ba805 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Thu, 6 Jun 2024 15:25:53 +0200
Subject: [PATCH 192/208] [Inference API] Make two fields looking like and used
 like constants static and final (#109425)

---
 .../inference/rest/RestDeleteInferenceEndpointAction.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceEndpointAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceEndpointAction.java
index 25280b8acf92d..287e286e95693 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceEndpointAction.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestDeleteInferenceEndpointAction.java
@@ -27,8 +27,8 @@
 @ServerlessScope(Scope.PUBLIC)
 public class RestDeleteInferenceEndpointAction extends BaseRestHandler {
 
-    private String FORCE_DELETE_NAME = "force";
-    private String DRY_RUN_NAME = "dry_run";
+    private static final String FORCE_DELETE_NAME = "force";
+    private static final String DRY_RUN_NAME = "dry_run";
 
     @Override
     public String getName() {

From aec07971d2cff071662f847745b12d2af6ed6de6 Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Thu, 6 Jun 2024 09:26:07 -0400
Subject: [PATCH 193/208] Mute
 org.elasticsearch.search.rank.RankFeatureShardPhaseTests.testProcessFetch
 #109428 (#109430)

related to: #109428
---
 muted-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/muted-tests.yml b/muted-tests.yml
index 32128da9c3712..d07d99e8e15ab 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -59,6 +59,9 @@ tests:
 - class: org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppendTests
   method: testEvaluateBlockWithoutNulls {TestCase=<cartesian_shape>, <cartesian_shape>}
   issue: https://github.com/elastic/elasticsearch/issues/109409
+- class: org.elasticsearch.search.rank.RankFeatureShardPhaseTests
+  method: testProcessFetch
+  issue: https://github.com/elastic/elasticsearch/issues/109428
 
 # Examples:
 #

From 8619713936bd77f938c48c7eeeaf28630a391b26 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Thu, 6 Jun 2024 15:30:47 +0200
Subject: [PATCH 194/208] [Inference API] Make ServiceUtils final and add
 private constructor (#109426)

---
 .../elasticsearch/xpack/inference/services/ServiceUtils.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java
index 4b5ec48f99b74..6d4b6d35323fd 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java
@@ -39,7 +39,7 @@
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY;
 
-public class ServiceUtils {
+public final class ServiceUtils {
     /**
      * Remove the object from the map and cast to the expected type.
      * If the object cannot be cast to type an ElasticsearchStatusException
@@ -624,4 +624,6 @@ public static SecureString apiKey(@Nullable ApiKeySecrets secrets) {
         // To avoid a possible null pointer throughout the code we'll create a noop api key of an empty array
         return secrets == null ? new SecureString(new char[0]) : secrets.apiKey();
     }
+
+    private ServiceUtils() {}
 }

From e70c77069609aa8f8fec037e690431ce3e402440 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Thu, 6 Jun 2024 16:30:24 +0200
Subject: [PATCH 195/208] [Inference API] Make http Utils final and add private
 constructor (#109427)

---
 .../elasticsearch/xpack/inference/external/http/Utils.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/Utils.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/Utils.java
index d82ed47441442..d61a3cbde48c5 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/Utils.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/Utils.java
@@ -22,7 +22,7 @@
 
 import static org.elasticsearch.core.Strings.format;
 
-public class Utils {
+public final class Utils {
 
     public static String getUrl(MockWebServer webServer) {
         return format("http://%s:%s", webServer.getHostName(), webServer.getPort());
@@ -46,4 +46,6 @@ public static Map<String, Object> entityAsMap(InputStream body) throws IOExcepti
             return parser.map();
         }
     }
+
+    private Utils() {}
 }

From d4d5d9320cecb8f6d70b81cda5bef81cd3cfc101 Mon Sep 17 00:00:00 2001
From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com>
Date: Thu, 6 Jun 2024 16:31:12 +0200
Subject: [PATCH 196/208] Fix semantic_text retrievers docs example (#109433)

---
 docs/reference/query-dsl/semantic-query.asciidoc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/reference/query-dsl/semantic-query.asciidoc b/docs/reference/query-dsl/semantic-query.asciidoc
index ccd94f0ecb77a..f3eb4b6ea68d3 100644
--- a/docs/reference/query-dsl/semantic-query.asciidoc
+++ b/docs/reference/query-dsl/semantic-query.asciidoc
@@ -100,9 +100,11 @@ GET my-index/_search
           }
         },
         {
-          "semantic": {
-            "field": "semantic_field",
-            "query": "shoes"
+          "standard": {
+            "semantic": {
+              "field": "semantic_field",
+              "query": "shoes"
+            }
           }
         }
       ],

From d89dae2a32357866e0907ffd58df852771bae529 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?=
 <istvan.szabo@elastic.co>
Date: Thu, 6 Jun 2024 16:45:46 +0200
Subject: [PATCH 197/208] [DOCS] Modifies semantic search-related docs to refer
 to the `semantic_text` workflow (#109418)

Co-authored-by: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com>
---
 .../semantic-search-elser.asciidoc            |  2 ++
 .../semantic-search-inference.asciidoc        |  8 +++---
 .../search-your-data/semantic-search.asciidoc | 26 +++++++++++--------
 3 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc
index c3eefec86e6f3..e7f503a4a6c70 100644
--- a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc
+++ b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc
@@ -13,6 +13,8 @@ query.
 The instructions in this tutorial shows you how to use ELSER to perform semantic
 search on your data.
 
+IMPORTANT: For the easiest way to perform semantic search in the {stack}, refer to the <<semantic-search-semantic-text, `semantic_text`>> end-to-end tutorial.
+
 NOTE: Only the first 512 extracted tokens per field are considered during
 semantic search with ELSER. Refer to
 {ml-docs}/ml-nlp-limitations.html#ml-nlp-elser-v1-limit-512[this page] for more
diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
index e53d895882b4e..6ecfea0a02dbc 100644
--- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
+++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
@@ -5,10 +5,12 @@
 <titleabbrev>Semantic search with the {infer} API</titleabbrev>
 ++++
 
-The instructions in this tutorial shows you how to use the {infer} API with various services to perform semantic search on your data.
+The instructions in this tutorial shows you how to use the {infer} API workflow with various services to perform semantic search on your data.
+
+IMPORTANT: For the easiest way to perform semantic search in the {stack}, refer to the <<semantic-search-semantic-text, `semantic_text`>> end-to-end tutorial.
+
 The following examples use Cohere's `embed-english-v3.0` model, the `all-mpnet-base-v2` model from HuggingFace, and OpenAI's `text-embedding-ada-002` second generation embedding model.
-You can use any Cohere and OpenAI models, they are all supported by the
-{infer} API.
+You can use any Cohere and OpenAI models, they are all supported by the {infer} API.
 For a list of supported models available on HuggingFace, refer to
 <<inference-example-hugging-face-supported-models, the supported model list>>.
 
diff --git a/docs/reference/search/search-your-data/semantic-search.asciidoc b/docs/reference/search/search-your-data/semantic-search.asciidoc
index 7dd7fbaf9823c..2d776077e13c5 100644
--- a/docs/reference/search/search-your-data/semantic-search.asciidoc
+++ b/docs/reference/search/search-your-data/semantic-search.asciidoc
@@ -1,25 +1,29 @@
 [[semantic-search]]
 == Semantic search
 
-Semantic search is a search method that helps you find data based on the intent
-and contextual meaning of a search query, instead of a match on query terms
+Semantic search is a search method that helps you find data based on the intent and contextual meaning of a search query, instead of a match on query terms
 (lexical search).
 
-{es} provides semantic search capabilities using {ml-docs}/ml-nlp.html[natural
-language processing (NLP)] and vector search. Deploying an NLP model to {es}
-enables it to extract text embeddings out of text. Embeddings are vectors that
-provide a numeric representation of a text. Pieces of content with similar
-meaning have similar representations. 
+{es} provides various semantic search capabilities using {ml-docs}/ml-nlp.html[natural language processing (NLP)] and vector search.
+Using an NLP model enables you to extract text embeddings out of text.
+Embeddings are vectors that provide a numeric representation of a text.
+Pieces of content with similar meaning have similar representations. 
+NLP models can be used in the {stack} various ways, you can:
+
+* deploy models in {es}
+* use the <<semantic-search-semantic-text, `semantic_text` workflow>> (recommended)
+* use the <<semantic-search-inference, {infer} API workflow>>
+
 
 [[semantic-search-diagram]]
 .A simplified representation of encoding textual concepts as vectors
 image::images/search/vector-search-oversimplification.png[A simplified representation of encoding textual concepts as vectors,align="center"]
 
-At query time, {es} can use the same NLP model to convert a query into
-embeddings, enabling you to find documents with similar text embeddings.
+At query time, {es} can use the same NLP model to convert a query into embeddings, enabling you to find documents with similar text embeddings.
+
+This guide shows you how to implement semantic search with {es}: From selecting an NLP model, to writing queries.
 
-This guide shows you how to implement semantic search with {es}, from selecting
-an NLP model, to writing queries.
+IMPORTANT: For the easiest way to perform semantic search in the {stack}, refer to the <<semantic-search-semantic-text, `semantic_text`>> end-to-end tutorial.
 
 [discrete]
 [[semantic-search-select-nlp-model]]

From 2822fa407218fdf566bcc33424d44c9f44c2d73a Mon Sep 17 00:00:00 2001
From: Sangmin Lee <55132820+snurf198@users.noreply.github.com>
Date: Fri, 7 Jun 2024 00:12:01 +0900
Subject: [PATCH 198/208] HFM typo fixed and test fixed (#108804)

<!-- Thank you for your interest in and contributing to Elasticsearch!
There are a few simple things to check before submitting your pull
request that can help with the review process. You should delete these
items from your submission, but they are here to help bring them to your
attention. -->

Fix typo in HistogramFieldMapper  and fixed related test code

closes https://github.com/elastic/elasticsearch/issues/108050
---
 .../xpack/analytics/mapper/HistogramFieldMapper.java            | 2 +-
 .../xpack/analytics/mapper/HistogramFieldMapperTests.java       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
index c7736a54187f1..cdcc0e495582a 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
@@ -410,7 +410,7 @@ public void parse(DocumentParserContext context) throws IOException {
                         + name()
                         + "] of type ["
                         + typeName()
-                        + "] doesn't not support indexing multiple values for the same field in the same document"
+                        + "] doesn't support indexing multiple values for the same field in the same document"
                 );
             }
             context.doc().addWithKey(fieldType().name(), field);
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java
index 2892ada15fec9..5e2bdaf2d465e 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java
@@ -87,7 +87,7 @@ public void testParseArrayValue() throws Exception {
         })));
         assertThat(
             e.getCause().getMessage(),
-            containsString("doesn't not support indexing multiple values for the same field in the same document")
+            containsString("doesn't support indexing multiple values for the same field in the same document")
         );
     }
 

From f6588ea29bafde02d09524d77761468e2bf27b9f Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Thu, 6 Jun 2024 17:29:06 +0100
Subject: [PATCH 199/208] Add parent filter query and BitsetProducer in
 NestedObjectMapper context (#109417)

---
 .../index/mapper/NestedObjectMapper.java      | 86 +++++++++++++++----
 .../FieldAliasMapperValidationTests.java      |  4 +-
 .../index/mapper/NestedLookupTests.java       |  4 +-
 .../index/mapper/NestedObjectMapperTests.java | 46 ++++++++--
 .../bucket/nested/NestedAggregatorTests.java  |  4 +-
 .../search/sort/AbstractSortTestCase.java     |  6 +-
 .../mapper/SemanticTextFieldMapper.java       | 25 ++++--
 7 files changed, 137 insertions(+), 38 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
index e98b8ba7deba1..65748847406ea 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
@@ -9,7 +9,9 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
 import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
@@ -18,6 +20,7 @@
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
  * A Mapper for nested objects
@@ -31,10 +34,12 @@ public static class Builder extends ObjectMapper.Builder {
         private Explicit<Boolean> includeInRoot = Explicit.IMPLICIT_FALSE;
         private Explicit<Boolean> includeInParent = Explicit.IMPLICIT_FALSE;
         private final IndexVersion indexCreatedVersion;
+        private final Function<Query, BitSetProducer> bitsetProducer;
 
-        public Builder(String name, IndexVersion indexCreatedVersion) {
+        public Builder(String name, IndexVersion indexCreatedVersion, Function<Query, BitSetProducer> bitSetProducer) {
             super(name, Explicit.IMPLICIT_TRUE);
             this.indexCreatedVersion = indexCreatedVersion;
+            this.bitsetProducer = bitSetProducer;
         }
 
         Builder includeInRoot(boolean includeInRoot) {
@@ -50,24 +55,21 @@ Builder includeInParent(boolean includeInParent) {
         @Override
         public NestedObjectMapper build(MapperBuilderContext context) {
             boolean parentIncludedInRoot = this.includeInRoot.value();
+            final Query parentTypeFilter;
             if (context instanceof NestedMapperBuilderContext nc) {
                 // we're already inside a nested mapper, so adjust our includes
                 if (nc.parentIncludedInRoot && this.includeInParent.value()) {
                     this.includeInRoot = Explicit.IMPLICIT_FALSE;
                 }
+                parentTypeFilter = nc.nestedTypeFilter;
             } else {
                 // this is a top-level nested mapper, so include_in_parent = include_in_root
                 parentIncludedInRoot |= this.includeInParent.value();
                 if (this.includeInParent.value()) {
                     this.includeInRoot = Explicit.IMPLICIT_FALSE;
                 }
+                parentTypeFilter = Queries.newNonNestedFilter(indexCreatedVersion);
             }
-            NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext(
-                context.buildFullName(name()),
-                parentIncludedInRoot,
-                context.getDynamic(dynamic),
-                context.getMergeReason()
-            );
             final String fullPath = context.buildFullName(name());
             final String nestedTypePath;
             if (indexCreatedVersion.before(IndexVersions.V_8_0_0)) {
@@ -75,6 +77,14 @@ public NestedObjectMapper build(MapperBuilderContext context) {
             } else {
                 nestedTypePath = fullPath;
             }
+            final Query nestedTypeFilter = NestedPathFieldMapper.filter(indexCreatedVersion, nestedTypePath);
+            NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext(
+                context.buildFullName(name()),
+                nestedTypeFilter,
+                parentIncludedInRoot,
+                context.getDynamic(dynamic),
+                context.getMergeReason()
+            );
             return new NestedObjectMapper(
                 name(),
                 fullPath,
@@ -83,8 +93,10 @@ public NestedObjectMapper build(MapperBuilderContext context) {
                 dynamic,
                 includeInParent,
                 includeInRoot,
+                parentTypeFilter,
                 nestedTypePath,
-                NestedPathFieldMapper.filter(indexCreatedVersion, nestedTypePath)
+                nestedTypeFilter,
+                bitsetProducer
             );
         }
     }
@@ -96,7 +108,11 @@ public Mapper.Builder parse(String name, Map<String, Object> node, MappingParser
             if (parseSubobjects(node).explicit()) {
                 throw new MapperParsingException("Nested type [" + name + "] does not support [subobjects] parameter");
             }
-            NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(name, parserContext.indexVersionCreated());
+            NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(
+                name,
+                parserContext.indexVersionCreated(),
+                parserContext::bitSetProducer
+            );
             parseNested(name, node, builder);
             parseObjectFields(node, parserContext, builder);
             return builder;
@@ -119,24 +135,43 @@ protected static void parseNested(String name, Map<String, Object> node, NestedO
     }
 
     private static class NestedMapperBuilderContext extends MapperBuilderContext {
-
         final boolean parentIncludedInRoot;
-
-        NestedMapperBuilderContext(String path, boolean parentIncludedInRoot, Dynamic dynamic, MapperService.MergeReason mergeReason) {
+        final Query nestedTypeFilter;
+
+        NestedMapperBuilderContext(
+            String path,
+            Query nestedTypeFilter,
+            boolean parentIncludedInRoot,
+            Dynamic dynamic,
+            MapperService.MergeReason mergeReason
+        ) {
             super(path, false, false, false, dynamic, mergeReason);
             this.parentIncludedInRoot = parentIncludedInRoot;
+            this.nestedTypeFilter = nestedTypeFilter;
         }
 
         @Override
         public MapperBuilderContext createChildContext(String name, Dynamic dynamic) {
-            return new NestedMapperBuilderContext(buildFullName(name), parentIncludedInRoot, getDynamic(dynamic), getMergeReason());
+            return new NestedMapperBuilderContext(
+                buildFullName(name),
+                nestedTypeFilter,
+                parentIncludedInRoot,
+                getDynamic(dynamic),
+                getMergeReason()
+            );
         }
     }
 
     private final Explicit<Boolean> includeInRoot;
     private final Explicit<Boolean> includeInParent;
+    // The query to identify parent documents
+    private final Query parentTypeFilter;
+    // The path of the nested field
     private final String nestedTypePath;
+    // The query to identify nested documents at this level
     private final Query nestedTypeFilter;
+    // Function to create a bitset for identifying parent documents
+    private final Function<Query, BitSetProducer> bitsetProducer;
 
     NestedObjectMapper(
         String name,
@@ -146,14 +181,22 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) {
         ObjectMapper.Dynamic dynamic,
         Explicit<Boolean> includeInParent,
         Explicit<Boolean> includeInRoot,
+        Query parentTypeFilter,
         String nestedTypePath,
-        Query nestedTypeFilter
+        Query nestedTypeFilter,
+        Function<Query, BitSetProducer> bitsetProducer
     ) {
         super(name, fullPath, enabled, Explicit.IMPLICIT_TRUE, Explicit.IMPLICIT_FALSE, dynamic, mappers);
+        this.parentTypeFilter = parentTypeFilter;
         this.nestedTypePath = nestedTypePath;
         this.nestedTypeFilter = nestedTypeFilter;
         this.includeInParent = includeInParent;
         this.includeInRoot = includeInRoot;
+        this.bitsetProducer = bitsetProducer;
+    }
+
+    public Query parentTypeFilter() {
+        return parentTypeFilter;
     }
 
     public Query nestedTypeFilter() {
@@ -177,13 +220,17 @@ public boolean isIncludeInRoot() {
         return this.includeInRoot.value();
     }
 
+    public Function<Query, BitSetProducer> bitsetProducer() {
+        return bitsetProducer;
+    }
+
     public Map<String, Mapper> getChildren() {
         return this.mappers;
     }
 
     @Override
     public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) {
-        NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(simpleName(), indexVersionCreated);
+        NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(simpleName(), indexVersionCreated, bitsetProducer);
         builder.enabled = enabled;
         builder.dynamic = dynamic;
         builder.includeInRoot = includeInRoot;
@@ -201,8 +248,10 @@ NestedObjectMapper withoutMappers() {
             dynamic,
             includeInParent,
             includeInRoot,
+            parentTypeFilter,
             nestedTypePath,
-            nestedTypeFilter
+            nestedTypeFilter,
+            bitsetProducer
         );
     }
 
@@ -270,8 +319,10 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex
             mergeResult.dynamic(),
             incInParent,
             incInRoot,
+            parentTypeFilter,
             nestedTypePath,
-            nestedTypeFilter
+            nestedTypeFilter,
+            bitsetProducer
         );
     }
 
@@ -285,6 +336,7 @@ protected MapperMergeContext createChildContext(MapperMergeContext mapperMergeCo
         return mapperMergeContext.createChildContext(
             new NestedMapperBuilderContext(
                 mapperBuilderContext.buildFullName(name),
+                nestedTypeFilter,
                 parentIncludedInRoot,
                 mapperBuilderContext.getDynamic(dynamic),
                 mapperBuilderContext.getMergeReason()
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java
index 8f6565cc5da94..886b0aa9e425d 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java
@@ -176,7 +176,9 @@ private static ObjectMapper createObjectMapper(String name) {
     }
 
     private static NestedObjectMapper createNestedObjectMapper(String name) {
-        return new NestedObjectMapper.Builder(name, IndexVersion.current()).build(MapperBuilderContext.root(false, false));
+        return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build(
+            MapperBuilderContext.root(false, false)
+        );
     }
 
     private static MappingLookup createMappingLookup(
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java
index 80ba37d8066b2..5c2fa6e89b0c6 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java
@@ -64,7 +64,9 @@ public void testMultiLevelParents() throws IOException {
     }
 
     private static NestedObjectMapper buildMapper(String name) {
-        return new NestedObjectMapper.Builder(name, IndexVersion.current()).build(MapperBuilderContext.root(false, false));
+        return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build(
+            MapperBuilderContext.root(false, false)
+        );
     }
 
     public void testAllParentFilters() {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
index 25e4ccdf4d3a9..412077b659b98 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.index.IndexVersion;
@@ -1500,12 +1501,12 @@ public void testIndexTemplatesMergeIncludes() throws IOException {
     }
 
     public void testMergeNested() {
-        NestedObjectMapper firstMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current()).includeInParent(true)
-            .includeInRoot(true)
-            .build(MapperBuilderContext.root(false, false));
-        NestedObjectMapper secondMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current()).includeInParent(false)
-            .includeInRoot(true)
-            .build(MapperBuilderContext.root(false, false));
+        NestedObjectMapper firstMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> {
+            throw new UnsupportedOperationException();
+        }).includeInParent(true).includeInRoot(true).build(MapperBuilderContext.root(false, false));
+        NestedObjectMapper secondMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> {
+            throw new UnsupportedOperationException();
+        }).includeInParent(false).includeInRoot(true).build(MapperBuilderContext.root(false, false));
 
         MapperException e = expectThrows(
             MapperException.class,
@@ -1533,6 +1534,39 @@ public void testWithoutMappers() throws IOException {
         assertThat(object.withoutMappers().toString(), equalTo(shallowObject.toString()));
     }
 
+    public void testNestedMapperFilters() throws Exception {
+        DocumentMapper docMapper = createDocumentMapper(mapping(b -> {
+            b.startObject("nested1");
+            {
+                b.field("type", "nested");
+                b.startObject("properties");
+                {
+                    b.startObject("field1").field("type", "text").endObject();
+                    b.startObject("sub_nested");
+                    {
+                        b.field("type", "nested");
+                        b.startObject("properties");
+                        {
+                            b.startObject("field2").field("type", "text").endObject();
+                        }
+                        b.endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+
+        assertThat(docMapper.mappers().nestedLookup().getNestedMappers().size(), equalTo(2));
+        assertThat(docMapper.mappers().nestedLookup().getNestedMappers().get("nested1"), instanceOf(NestedObjectMapper.class));
+        NestedObjectMapper mapper1 = docMapper.mappers().nestedLookup().getNestedMappers().get("nested1");
+        assertThat(mapper1.parentTypeFilter(), equalTo(Queries.newNonNestedFilter(IndexVersion.current())));
+
+        NestedObjectMapper mapper2 = docMapper.mappers().nestedLookup().getNestedMappers().get("nested1.sub_nested");
+        assertThat(mapper2.parentTypeFilter(), equalTo(mapper1.nestedTypeFilter()));
+    }
+
     private NestedObjectMapper createNestedObjectMapperWithAllParametersSet(CheckedConsumer<XContentBuilder, IOException> propertiesBuilder)
         throws IOException {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
index 09c13a96da704..4ec2e5ab49cd3 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
@@ -912,6 +912,8 @@ protected List<ObjectMapper> objectMappers() {
     );
 
     public static NestedObjectMapper nestedObject(String path) {
-        return new NestedObjectMapper.Builder(path, IndexVersion.current()).build(MapperBuilderContext.root(false, false));
+        return new NestedObjectMapper.Builder(path, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build(
+            MapperBuilderContext.root(false, false)
+        );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index 5fcd4eeeb2636..f5fbca13db1db 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -194,9 +194,9 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea
             IndexFieldData.Builder builder = fieldType.fielddataBuilder(fdc);
             return builder.build(new IndexFieldDataCache.None(), null);
         };
-        NestedLookup nestedLookup = NestedLookup.build(
-            List.of(new NestedObjectMapper.Builder("path", IndexVersion.current()).build(MapperBuilderContext.root(false, false)))
-        );
+        NestedLookup nestedLookup = NestedLookup.build(List.of(new NestedObjectMapper.Builder("path", IndexVersion.current(), query -> {
+            throw new UnsupportedOperationException();
+        }).build(MapperBuilderContext.root(false, false))));
         return new SearchExecutionContext(
             0,
             0,
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
index 6874938f1e118..355f52e5e3cdc 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference.mapper;
 
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.common.Explicit;
@@ -78,7 +79,7 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
     public static final String CONTENT_TYPE = "semantic_text";
 
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.indexVersionCreated()),
+        (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer),
         List.of(notInMultiFields(CONTENT_TYPE), notFromDynamicTemplates(CONTENT_TYPE))
     );
 
@@ -110,10 +111,10 @@ public static class Builder extends FieldMapper.Builder {
 
         private Function<MapperBuilderContext, ObjectMapper> inferenceFieldBuilder;
 
-        public Builder(String name, IndexVersion indexVersionCreated) {
+        public Builder(String name, IndexVersion indexVersionCreated, Function<Query, BitSetProducer> bitSetProducer) {
             super(name);
             this.indexVersionCreated = indexVersionCreated;
-            this.inferenceFieldBuilder = c -> createInferenceField(c, indexVersionCreated, modelSettings.get());
+            this.inferenceFieldBuilder = c -> createInferenceField(c, indexVersionCreated, modelSettings.get(), bitSetProducer);
         }
 
         public Builder setInferenceId(String id) {
@@ -181,7 +182,7 @@ public Iterator<Mapper> iterator() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), fieldType().indexVersionCreated).init(this);
+        return new Builder(simpleName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer()).init(this);
     }
 
     @Override
@@ -219,7 +220,11 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         final SemanticTextFieldMapper mapper;
         if (fieldType().getModelSettings() == null) {
             context.path().remove();
-            Builder builder = (Builder) new Builder(simpleName(), fieldType().indexVersionCreated).init(this);
+            Builder builder = (Builder) new Builder(
+                simpleName(),
+                fieldType().indexVersionCreated,
+                fieldType().getChunksField().bitsetProducer()
+            ).init(this);
             try {
                 mapper = builder.setModelSettings(field.inference().modelSettings())
                     .setInferenceId(field.inference().inferenceId())
@@ -441,18 +446,20 @@ public QueryBuilder semanticQuery(InferenceResults inferenceResults, float boost
     private static ObjectMapper createInferenceField(
         MapperBuilderContext context,
         IndexVersion indexVersionCreated,
-        @Nullable SemanticTextField.ModelSettings modelSettings
+        @Nullable SemanticTextField.ModelSettings modelSettings,
+        Function<Query, BitSetProducer> bitSetProducer
     ) {
         return new ObjectMapper.Builder(INFERENCE_FIELD, Explicit.EXPLICIT_TRUE).dynamic(ObjectMapper.Dynamic.FALSE)
-            .add(createChunksField(indexVersionCreated, modelSettings))
+            .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer))
             .build(context);
     }
 
     private static NestedObjectMapper.Builder createChunksField(
         IndexVersion indexVersionCreated,
-        SemanticTextField.ModelSettings modelSettings
+        @Nullable SemanticTextField.ModelSettings modelSettings,
+        Function<Query, BitSetProducer> bitSetProducer
     ) {
-        NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder(CHUNKS_FIELD, indexVersionCreated);
+        NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder(CHUNKS_FIELD, indexVersionCreated, bitSetProducer);
         chunksField.dynamic(ObjectMapper.Dynamic.FALSE);
         KeywordFieldMapper.Builder chunkTextField = new KeywordFieldMapper.Builder(CHUNKED_TEXT_FIELD, indexVersionCreated).indexed(false)
             .docValues(false);

From 25b77ea36686e1b94aa44367e8b89e71b1f084bc Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Thu, 6 Jun 2024 18:33:45 +0200
Subject: [PATCH 200/208] Use System::nanoTime as default time source in
 SharedBlobCacheService (#109420)

We need to use `System#nanoTime` here. We use this time source for
measuring the timing of blob IO. The 200ms default accuracy isn't enough
here, we need accurate timing.
---
 .../elasticsearch/blobcache/shared/SharedBlobCacheService.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
index c3cd60d88c6f2..ac22d22d5affb 100644
--- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
@@ -335,7 +335,7 @@ public SharedBlobCacheService(
         String ioExecutor,
         BlobCacheMetrics blobCacheMetrics
     ) {
-        this(environment, settings, threadPool, ioExecutor, blobCacheMetrics, threadPool::relativeTimeInNanos);
+        this(environment, settings, threadPool, ioExecutor, blobCacheMetrics, System::nanoTime);
     }
 
     public SharedBlobCacheService(

From 5fc04bacd6dacb9ca57403b198534ba9600bad18 Mon Sep 17 00:00:00 2001
From: Mark Vieira <portugee@gmail.com>
Date: Thu, 6 Jun 2024 09:34:07 -0700
Subject: [PATCH 201/208] Add publish plugin to ES|QL projects (#109441)

We want to make the JavaDocs for ES|QL publicly available. Add the
publishing plugin so we can publish these artifacts.
---
 x-pack/plugin/esql-core/build.gradle        | 1 +
 x-pack/plugin/esql/build.gradle             | 2 ++
 x-pack/plugin/esql/compute/ann/build.gradle | 5 +++++
 x-pack/plugin/esql/compute/build.gradle     | 5 +++++
 4 files changed, 13 insertions(+)

diff --git a/x-pack/plugin/esql-core/build.gradle b/x-pack/plugin/esql-core/build.gradle
index ed9f5066d10e7..796ec1d137155 100644
--- a/x-pack/plugin/esql-core/build.gradle
+++ b/x-pack/plugin/esql-core/build.gradle
@@ -1,5 +1,6 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-test-artifact'
+apply plugin: 'elasticsearch.publish'
 
 esplugin {
   name 'x-pack-esql-core'
diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle
index faa5a118a90cd..92071543aa27e 100644
--- a/x-pack/plugin/esql/build.gradle
+++ b/x-pack/plugin/esql/build.gradle
@@ -4,6 +4,8 @@ import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask;
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.string-templates'
+apply plugin: 'elasticsearch.publish'
+
 esplugin {
   name 'x-pack-esql'
   description 'The plugin that powers ESQL for Elasticsearch'
diff --git a/x-pack/plugin/esql/compute/ann/build.gradle b/x-pack/plugin/esql/compute/ann/build.gradle
index ee8d8c62dff39..072f1f6628b07 100644
--- a/x-pack/plugin/esql/compute/ann/build.gradle
+++ b/x-pack/plugin/esql/compute/ann/build.gradle
@@ -1,4 +1,9 @@
 apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.publish'
+
+base {
+  archivesName = 'x-pack-esql-compute-ann'
+}
 
 tasks.named('forbiddenApisMain').configure {
   // doesn't depend on anything
diff --git a/x-pack/plugin/esql/compute/build.gradle b/x-pack/plugin/esql/compute/build.gradle
index fce051ed60038..6092b006a551c 100644
--- a/x-pack/plugin/esql/compute/build.gradle
+++ b/x-pack/plugin/esql/compute/build.gradle
@@ -1,5 +1,10 @@
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.string-templates'
+apply plugin: 'elasticsearch.publish'
+
+base {
+  archivesName = 'x-pack-esql-compute'
+}
 
 dependencies {
   compileOnly project(':server')

From b92ff3cf2d4ca67157be487a8614f3aed72a81bf Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Thu, 6 Jun 2024 18:48:25 +0200
Subject: [PATCH 202/208] [Inference API] Deduplicate methods parsing a single
 float in several response entities (#109432)

---
 .../external/response/XContentUtils.java      | 24 ++++++++++
 .../CohereEmbeddingsResponseEntity.java       |  9 +---
 ...oogleAiStudioEmbeddingsResponseEntity.java |  9 +---
 .../HuggingFaceEmbeddingsResponseEntity.java  |  9 +---
 .../OpenAiEmbeddingsResponseEntity.java       |  9 +---
 .../external/response/XContentUtilsTests.java | 48 +++++++++++++++++++
 6 files changed, 80 insertions(+), 28 deletions(-)

diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/XContentUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/XContentUtils.java
index 55a7f35710cf6..a4f48510bc0e6 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/XContentUtils.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/XContentUtils.java
@@ -7,10 +7,13 @@
 
 package org.elasticsearch.xpack.inference.external.response;
 
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.core.Strings.format;
 
 public class XContentUtils {
@@ -74,5 +77,26 @@ public static void consumeUntilObjectEnd(XContentParser parser) throws IOExcepti
         }
     }
 
+    /**
+     * Parses a single float.
+     * In the context of the inference API this method is usually used in conjunction
+     * with {@link XContentParserUtils#parseList(XContentParser, CheckedFunction)} to parse a list of floats of an embedding:
+     *
+     * <pre>
+     *     <code>
+     *       var floats = XContentParserUtils.parseList(parser, XContentUtils::parseFloat);
+     *     </code>
+     * </pre>
+     *
+     * @param parser
+     * @return single float
+     * @throws IOException
+     */
+    public static float parseFloat(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
+        return parser.floatValue();
+    }
+
     private XContentUtils() {}
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java
index 88fe54c338bd5..ee4b43ec2e41e 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.response.XContentUtils;
 import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingType;
 
 import java.io.IOException;
@@ -219,15 +220,9 @@ private static InferenceServiceResults parseFloatEmbeddingsArray(XContentParser
 
     private static TextEmbeddingResults.Embedding parseFloatArrayEntry(XContentParser parser) throws IOException {
         ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
-        List<Float> embeddingValuesList = parseList(parser, CohereEmbeddingsResponseEntity::parseEmbeddingFloatEntry);
+        List<Float> embeddingValuesList = parseList(parser, XContentUtils::parseFloat);
         return TextEmbeddingResults.Embedding.of(embeddingValuesList);
     }
 
-    private static Float parseEmbeddingFloatEntry(XContentParser parser) throws IOException {
-        XContentParser.Token token = parser.currentToken();
-        ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
-        return parser.floatValue();
-    }
-
     private CohereEmbeddingsResponseEntity() {}
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java
index d214aea42fb06..69eefcefc6148 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.response.XContentUtils;
 
 import java.io.IOException;
 import java.util.List;
@@ -94,18 +95,12 @@ private static TextEmbeddingResults.Embedding parseEmbeddingObject(XContentParse
 
         positionParserAtTokenAfterField(parser, "values", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-        List<Float> embeddingValuesList = parseList(parser, GoogleAiStudioEmbeddingsResponseEntity::parseEmbeddingList);
+        List<Float> embeddingValuesList = parseList(parser, XContentUtils::parseFloat);
         // parse and discard the rest of the object
         consumeUntilObjectEnd(parser);
 
         return TextEmbeddingResults.Embedding.of(embeddingValuesList);
     }
 
-    private static float parseEmbeddingList(XContentParser parser) throws IOException {
-        XContentParser.Token token = parser.currentToken();
-        ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
-        return parser.floatValue();
-    }
-
     private GoogleAiStudioEmbeddingsResponseEntity() {}
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java
index 98c4ec7667fd0..7e2e3abd6275a 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.response.XContentUtils;
 
 import java.io.IOException;
 import java.util.List;
@@ -145,15 +146,9 @@ private static TextEmbeddingResults parseObjectFormat(XContentParser parser) thr
     private static TextEmbeddingResults.Embedding parseEmbeddingEntry(XContentParser parser) throws IOException {
         ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser);
 
-        List<Float> embeddingValuesList = parseList(parser, HuggingFaceEmbeddingsResponseEntity::parseEmbeddingList);
+        List<Float> embeddingValuesList = parseList(parser, XContentUtils::parseFloat);
         return TextEmbeddingResults.Embedding.of(embeddingValuesList);
     }
 
-    private static float parseEmbeddingList(XContentParser parser) throws IOException {
-        XContentParser.Token token = parser.currentToken();
-        ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
-        return parser.floatValue();
-    }
-
     private HuggingFaceEmbeddingsResponseEntity() {}
 }
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java
index 711944ea820a9..eeed83dfe69bd 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.inference.external.http.HttpResult;
 import org.elasticsearch.xpack.inference.external.request.Request;
+import org.elasticsearch.xpack.inference.external.response.XContentUtils;
 
 import java.io.IOException;
 import java.util.List;
@@ -98,18 +99,12 @@ private static TextEmbeddingResults.Embedding parseEmbeddingObject(XContentParse
 
         positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE);
 
-        List<Float> embeddingValuesList = parseList(parser, OpenAiEmbeddingsResponseEntity::parseEmbeddingList);
+        List<Float> embeddingValuesList = parseList(parser, XContentUtils::parseFloat);
         // parse and discard the rest of the object
         consumeUntilObjectEnd(parser);
 
         return TextEmbeddingResults.Embedding.of(embeddingValuesList);
     }
 
-    private static float parseEmbeddingList(XContentParser parser) throws IOException {
-        XContentParser.Token token = parser.currentToken();
-        ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser);
-        return parser.floatValue();
-    }
-
     private OpenAiEmbeddingsResponseEntity() {}
 }
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/XContentUtilsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/XContentUtilsTests.java
index 897c648eb942f..e1d786819a536 100644
--- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/XContentUtilsTests.java
+++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/XContentUtilsTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.inference.external.response;
 
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentEOFException;
 import org.elasticsearch.xcontent.XContentParser;
@@ -16,6 +17,7 @@
 import java.util.Locale;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class XContentUtilsTests extends ESTestCase {
 
@@ -233,4 +235,50 @@ public void testConsumeUntilObjectEnd_InArray() throws IOException {
             assertNull(parser.nextToken()); // fully parsed
         }
     }
+
+    public void testParseFloat_SingleFloatValue() throws IOException {
+        var json = """
+             {
+               "key": 1.23
+              }
+            """;
+        var errorFormat = "Error: %s";
+
+        try (XContentParser parser = createParser(XContentType.JSON.xContent(), json)) {
+            XContentUtils.positionParserAtTokenAfterField(parser, "key", errorFormat);
+            Float value = XContentUtils.parseFloat(parser);
+
+            assertThat(value, equalTo(1.23F));
+        }
+    }
+
+    public void testParseFloat_SingleIntValue() throws IOException {
+        var json = """
+             {
+               "key": 1
+             }
+            """;
+        var errorFormat = "Error: %s";
+
+        try (XContentParser parser = createParser(XContentType.JSON.xContent(), json)) {
+            XContentUtils.positionParserAtTokenAfterField(parser, "key", errorFormat);
+            Float value = XContentUtils.parseFloat(parser);
+
+            assertThat(value, equalTo(1.0F));
+        }
+    }
+
+    public void testParseFloat_ThrowsIfNotANumber() throws IOException {
+        var json = """
+             {
+               "key": "value"
+             }
+            """;
+        var errorFormat = "Error: %s";
+
+        try (XContentParser parser = createParser(XContentType.JSON.xContent(), json)) {
+            XContentUtils.positionParserAtTokenAfterField(parser, "key", errorFormat);
+            expectThrows(ParsingException.class, () -> XContentUtils.parseFloat(parser));
+        }
+    }
 }

From 4c17c861d257160027d5d1dbb20ab69307decff9 Mon Sep 17 00:00:00 2001
From: Benjamin Trent <ben.w.trent@gmail.com>
Date: Thu, 6 Jun 2024 14:20:47 -0400
Subject: [PATCH 203/208] Correct how hex strings are handled when dynamically
 updating vector dims (#109423)

closes: https://github.com/elastic/elasticsearch/issues/109411
---
 docs/changelog/109423.yaml                    |  5 +++
 ...70_knn_search_hex_encoded_byte_vectors.yml | 40 +++++++++++++++++++
 .../vectors/DenseVectorFieldMapper.java       | 22 ++++++++++
 3 files changed, 67 insertions(+)
 create mode 100644 docs/changelog/109423.yaml

diff --git a/docs/changelog/109423.yaml b/docs/changelog/109423.yaml
new file mode 100644
index 0000000000000..5f594ea482338
--- /dev/null
+++ b/docs/changelog/109423.yaml
@@ -0,0 +1,5 @@
+pr: 109423
+summary: Correct how hex strings are handled when dynamically updating vector dims
+area: Vector Search
+type: bug
+issues: []
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/170_knn_search_hex_encoded_byte_vectors.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/170_knn_search_hex_encoded_byte_vectors.yml
index 44f17e2269027..be1e619d046ac 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/170_knn_search_hex_encoded_byte_vectors.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/170_knn_search_hex_encoded_byte_vectors.yml
@@ -161,3 +161,43 @@ setup:
   - match: { hits.hits.0._id: "3" }
   - match: { hits.hits.1._id: "2" }
   - match: { hits.hits.2._id: "1" }
+---
+"Dynamic dimensions for hex-encoded string":
+  - requires:
+      cluster_features: "gte_v8.15.0"
+      reason: 'hex encoding for byte vectors fixed in 8.15'
+
+  - do:
+      indices.create:
+        index: knn_hex_vector_index_dyn_dims
+        body:
+          settings:
+            number_of_shards: 1
+          mappings:
+            properties:
+              my_vector_byte:
+                type: dense_vector
+                index : false
+                element_type: byte
+              my_vector_byte_indexed:
+                type: dense_vector
+                index: true
+                element_type: byte
+                similarity : l2_norm
+
+  # [-128, 127, 10] - is encoded as '807f0a'
+  - do:
+      index:
+        index: knn_hex_vector_index_dyn_dims
+        id: "1"
+        body:
+          my_vector_byte: "807f0a"
+          my_vector_byte_indexed: "807f0a"
+
+  # assert the index is created with 3 dimensions
+  - do:
+      indices.get_mapping:
+        index: knn_hex_vector_index_dyn_dims
+
+  - match: { knn_hex_vector_index_dyn_dims.mappings.properties.my_vector_byte.dims: 3 }
+  - match: { knn_hex_vector_index_dyn_dims.mappings.properties.my_vector_byte_indexed.dims: 3 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
index b47f6fd80a77e..c00f360e94c7e 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
@@ -467,6 +467,28 @@ int getNumBytes(int dimensions) {
             ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes) {
                 return ByteBuffer.wrap(new byte[numBytes]);
             }
+
+            @Override
+            int parseDimensionCount(DocumentParserContext context) throws IOException {
+                XContentParser.Token currentToken = context.parser().currentToken();
+                return switch (currentToken) {
+                    case START_ARRAY -> {
+                        int index = 0;
+                        for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
+                            index++;
+                        }
+                        yield index;
+                    }
+                    case VALUE_STRING -> {
+                        byte[] decodedVector = HexFormat.of().parseHex(context.parser().text());
+                        yield decodedVector.length;
+                    }
+                    default -> throw new ParsingException(
+                        context.parser().getTokenLocation(),
+                        format("Unsupported type [%s] for provided value [%s]", currentToken, context.parser().text())
+                    );
+                };
+            }
         },
 
         FLOAT(4) {

From bc1fcdef3ccea5808c0f52a5d41101a4dfa9df06 Mon Sep 17 00:00:00 2001
From: Panagiotis Bailis <pmpailis@gmail.com>
Date: Thu, 6 Jun 2024 23:11:59 +0300
Subject: [PATCH 204/208] Fix for duplicate key during map initialization
 (#109431)

---
 muted-tests.yml                                                | 3 ---
 .../elasticsearch/search/rank/RankFeatureShardPhaseTests.java  | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index d07d99e8e15ab..32128da9c3712 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -59,9 +59,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppendTests
   method: testEvaluateBlockWithoutNulls {TestCase=<cartesian_shape>, <cartesian_shape>}
   issue: https://github.com/elastic/elasticsearch/issues/109409
-- class: org.elasticsearch.search.rank.RankFeatureShardPhaseTests
-  method: testProcessFetch
-  issue: https://github.com/elastic/elasticsearch/issues/109428
 
 # Examples:
 #
diff --git a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
index cf464044cd701..a8425a1c59d9a 100644
--- a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
@@ -266,7 +266,7 @@ public void testPrepareForFetchWhileTaskIsCancelled() {
 
     public void testProcessFetch() {
         final String fieldName = "some_field";
-        int numDocs = randomIntBetween(10, 30);
+        int numDocs = randomIntBetween(15, 30);
         Map<Integer, String> expectedFieldData = Map.of(4, "doc_4_aardvark", 9, "doc_9_aardvark", numDocs - 1, "last_doc_aardvark");
 
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

From 5e6ffaae09f23a8de306675f0ccec0cc1d26f51f Mon Sep 17 00:00:00 2001
From: lixiaolong <qingkong_lxl@163.com>
Date: Fri, 7 Jun 2024 05:41:13 +0800
Subject: [PATCH 205/208] Ignore additional cpu.stat fields (#108019)

cpu.stat may contain additional fields depending on linux configuration. This commit ignores any fields past the 3 that Elasticsearch looks at.

closes #107983
---
 docs/changelog/108019.yaml                               | 6 ++++++
 .../main/java/org/elasticsearch/monitor/os/OsProbe.java  | 9 +++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)
 create mode 100644 docs/changelog/108019.yaml

diff --git a/docs/changelog/108019.yaml b/docs/changelog/108019.yaml
new file mode 100644
index 0000000000000..69e8e9fd371f8
--- /dev/null
+++ b/docs/changelog/108019.yaml
@@ -0,0 +1,6 @@
+pr: 108019
+summary: Ignore additional cpu.stat fields
+area: Infra/Core
+type: bug
+issues:
+  - 107983
diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
index 64fcc2f8ff684..130850640cf3c 100644
--- a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
+++ b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
@@ -469,8 +469,13 @@ private OsStats.Cgroup.CpuStat getCgroupCpuAcctCpuStat(final String controlGroup
      * nr_bursts \d+
      * burst_time
      * </pre></blockquote>
-     * These additional fields are currently ignored.
      *
+     * When schedstat_enabled is enabled, an additional statistics information {@code wait_sum} will also be available
+     * <blockquote><pre>
+     * wait_sum \d+
+     * </pre></blockquote>
+     * {@code wait_sum} represent the conflict between task groups, which is simply sum the wait time of group's cfs_rq
+     *  These three additional fields are currently ignored.
      * @param controlGroup the control group to which the Elasticsearch process belongs for the {@code cpu} subsystem
      * @return the lines from {@code cpu.stat}
      * @throws IOException if an I/O exception occurs reading {@code cpu.stat} for the control group
@@ -478,7 +483,7 @@ private OsStats.Cgroup.CpuStat getCgroupCpuAcctCpuStat(final String controlGroup
     @SuppressForbidden(reason = "access /sys/fs/cgroup/cpu")
     List<String> readSysFsCgroupCpuAcctCpuStat(final String controlGroup) throws IOException {
         final List<String> lines = Files.readAllLines(PathUtils.get("/sys/fs/cgroup/cpu", controlGroup, "cpu.stat"));
-        assert lines != null && (lines.size() == 3 || lines.size() == 5);
+        assert lines != null && (lines.size() >= 3);
         return lines;
     }
 

From 7916e6a231db48e73b7cb31889ba25d814c49d6d Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 6 Jun 2024 21:38:51 -0400
Subject: [PATCH 206/208] ESQL: Implement LOOKUP, an "inline" enrich (#107987)

This adds support for `LOOKUP`, a command that implements a sort of
inline `ENRICH`, using data that is passed in the request:

```
$ curl -uelastic:password -HContent-Type:application/json -XPOST \
    'localhost:9200/_query?error_trace&pretty&format=txt' \
-d'{
    "query": "ROW a=1::LONG | LOOKUP t ON a",
    "tables": {
        "t": {
            "a:long":     [    1,     4,     2],
            "v1:integer": [   10,    11,    12],
            "v2:keyword": ["cat", "dog", "wow"]
        }
    },
    "version": "2024.04.01"
}'
      v1       |      v2       |       a
---------------+---------------+---------------
10             |cat            |1
```

This required these PRs: * #107624 * #107634 * #107701 * #107762 *
#107923 * #107894 * #107982 * #108012 * #108020 * #108169 * #108191 *
#108334 * #108482 * #108696 * #109040 * #109045

Closes #107306
---
 docs/changelog/107987.yaml                    |   6 +
 docs/reference/esql/esql-commands.asciidoc    |   6 +
 docs/reference/esql/esql-query-api.asciidoc   |   5 +
 .../functions/kibana/definition/mv_zip.json   | 216 +++++++++++++++
 .../esql/functions/types/mv_zip.asciidoc      |  10 +
 .../esql/processing-commands/lookup.asciidoc  |  65 +++++
 .../xpack/esql/qa/rest/EsqlSpecTestCase.java  |  56 ++++
 .../xpack/esql/qa/rest/RestEsqlTestCase.java  |  10 +
 .../xpack/esql/EsqlTestUtils.java             | 121 +++++++--
 .../src/main/resources/lookup.csv-spec        | 253 ++++++++++++++++++
 .../org/elasticsearch/xpack/esql/Column.java  |   7 +-
 .../xpack/esql/action/EsqlCapabilities.java   |   6 +-
 .../xpack/esql/action/ParseTables.java        |  48 +++-
 .../xpack/esql/analysis/Analyzer.java         | 134 +++++++++-
 .../xpack/esql/analysis/Verifier.java         |  17 +-
 .../xpack/esql/io/stream/PlanNamedTypes.java  |  14 +-
 .../esql/io/stream/PlanStreamOutput.java      |   3 +-
 .../optimizer/LocalLogicalPlanOptimizer.java  |   3 +-
 .../esql/optimizer/LogicalPlanOptimizer.java  |  22 ++
 .../esql/optimizer/PhysicalPlanOptimizer.java |   8 +
 .../xpack/esql/parser/LogicalPlanBuilder.java |   3 +-
 .../xpack/esql/plan/logical/Lookup.java       | 154 +++++++++++
 .../xpack/esql/plan/logical/join/Join.java    | 175 ++++++++++++
 .../esql/plan/logical/join/JoinConfig.java    |  46 ++++
 .../esql/plan/logical/join/JoinType.java      |  52 ++++
 .../plan/logical/local/LocalRelation.java     |  15 ++
 .../esql/plan/physical/HashJoinExec.java      | 137 ++++++++++
 .../esql/plan/physical/LocalSourceExec.java   |  15 ++
 .../esql/planner/LocalExecutionPlanner.java   |  71 +++++
 .../xpack/esql/planner/Mapper.java            |  57 +++-
 .../esql/action/EsqlQueryRequestTests.java    |  29 ++
 .../xpack/esql/analysis/AnalyzerTests.java    |  91 ++++++-
 .../esql/io/stream/PlanNamedTypesTests.java   |  16 +-
 .../optimizer/LogicalPlanOptimizerTests.java  | 156 +++++++++++
 .../optimizer/PhysicalPlanOptimizerTests.java | 152 +++++++++++
 .../esql/parser/StatementParserTests.java     |  11 +
 .../TestPhysicalOperationProviders.java       |   1 +
 .../esql/tree/EsqlNodeSubclassTests.java      |  11 +-
 .../xpack/ql/rule/RuleExecutor.java           |   2 +-
 .../rest-api-spec/test/esql/150_lookup.yml    | 233 ++++++++++++++--
 40 files changed, 2383 insertions(+), 54 deletions(-)
 create mode 100644 docs/changelog/107987.yaml
 create mode 100644 docs/reference/esql/processing-commands/lookup.asciidoc
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinType.java
 create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/HashJoinExec.java

diff --git a/docs/changelog/107987.yaml b/docs/changelog/107987.yaml
new file mode 100644
index 0000000000000..e8afebde0b190
--- /dev/null
+++ b/docs/changelog/107987.yaml
@@ -0,0 +1,6 @@
+pr: 107987
+summary: "ESQL: Implement LOOKUP, an \"inline\" enrich"
+area: ES|QL
+type: enhancement
+issues:
+ - 107306
diff --git a/docs/reference/esql/esql-commands.asciidoc b/docs/reference/esql/esql-commands.asciidoc
index 1f07361b89aac..bed79299b1cc1 100644
--- a/docs/reference/esql/esql-commands.asciidoc
+++ b/docs/reference/esql/esql-commands.asciidoc
@@ -39,6 +39,9 @@ image::images/esql/processing-command.svg[A processing command changing an input
 * <<esql-grok>>
 * <<esql-keep>>
 * <<esql-limit>>
+ifeval::["{release-state}"=="unreleased"]
+* experimental:[] <<esql-lookup>>
+endif::[]
 * experimental:[] <<esql-mv_expand>>
 * <<esql-rename>>
 * <<esql-sort>>
@@ -58,6 +61,9 @@ include::processing-commands/eval.asciidoc[]
 include::processing-commands/grok.asciidoc[]
 include::processing-commands/keep.asciidoc[]
 include::processing-commands/limit.asciidoc[]
+ifeval::["{release-state}"=="unreleased"]
+include::processing-commands/lookup.asciidoc[]
+endif::[]
 include::processing-commands/mv_expand.asciidoc[]
 include::processing-commands/rename.asciidoc[]
 include::processing-commands/sort.asciidoc[]
diff --git a/docs/reference/esql/esql-query-api.asciidoc b/docs/reference/esql/esql-query-api.asciidoc
index fbac57d88118e..2cdd97ceab176 100644
--- a/docs/reference/esql/esql-query-api.asciidoc
+++ b/docs/reference/esql/esql-query-api.asciidoc
@@ -78,6 +78,11 @@ For syntax, refer to <<esql-locale-param>>.
 `query`::
 (Required, string) {esql} query to run. For syntax, refer to <<esql-syntax>>.
 
+ifeval::["{release-state}"=="unreleased"]
+`table`::
+(Optional, object) Named "table" parameters that can be referenced by the <<esql-lookup>> command.
+endif::[]
+
 [discrete]
 [role="child_attributes"]
 [[esql-query-api-response-body]]
diff --git a/docs/reference/esql/functions/kibana/definition/mv_zip.json b/docs/reference/esql/functions/kibana/definition/mv_zip.json
index 94c13f14c43e5..b6e5c86a3f0b8 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_zip.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_zip.json
@@ -17,6 +17,222 @@
           "type" : "keyword",
           "optional" : false,
           "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string1",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "string2",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "delim",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "Delimiter. Optional; if omitted, `,` is used as a default delimiter."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string1",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "string2",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "delim",
+          "type" : "text",
+          "optional" : true,
+          "description" : "Delimiter. Optional; if omitted, `,` is used as a default delimiter."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string1",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "string2",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string1",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "string2",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "delim",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "Delimiter. Optional; if omitted, `,` is used as a default delimiter."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string1",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "string2",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "delim",
+          "type" : "text",
+          "optional" : true,
+          "description" : "Delimiter. Optional; if omitted, `,` is used as a default delimiter."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string1",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "string2",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string1",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "string2",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "delim",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "Delimiter. Optional; if omitted, `,` is used as a default delimiter."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string1",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "string2",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "delim",
+          "type" : "text",
+          "optional" : true,
+          "description" : "Delimiter. Optional; if omitted, `,` is used as a default delimiter."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string1",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "string2",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
+    {
+      "params" : [
+        {
+          "name" : "string1",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        },
+        {
+          "name" : "string2",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Multivalue expression."
         },
         {
           "name" : "delim",
diff --git a/docs/reference/esql/functions/types/mv_zip.asciidoc b/docs/reference/esql/functions/types/mv_zip.asciidoc
index 514041202a1d5..5e3e1b57d6a55 100644
--- a/docs/reference/esql/functions/types/mv_zip.asciidoc
+++ b/docs/reference/esql/functions/types/mv_zip.asciidoc
@@ -6,5 +6,15 @@
 |===
 string1 | string2 | delim | result
 keyword | keyword | keyword | keyword
+keyword | keyword | text | keyword
+keyword | keyword | | keyword
+keyword | text | keyword | keyword
+keyword | text | text | keyword
+keyword | text | | keyword
+text | keyword | keyword | keyword
+text | keyword | text | keyword
+text | keyword | | keyword
+text | text | keyword | keyword
 text | text | text | keyword
+text | text | | keyword
 |===
diff --git a/docs/reference/esql/processing-commands/lookup.asciidoc b/docs/reference/esql/processing-commands/lookup.asciidoc
new file mode 100644
index 0000000000000..5507c6cc29d1b
--- /dev/null
+++ b/docs/reference/esql/processing-commands/lookup.asciidoc
@@ -0,0 +1,65 @@
+[discrete]
+[[esql-lookup]]
+=== `LOOKUP`
+
+experimental::["LOOKUP is a highly experimental and only available in SNAPSHOT versions."]
+
+**Syntax**
+
+[source,esql]
+----
+LOOKUP table ON match_field1[, match_field2, ...]
+----
+
+*Parameters*
+
+`table`::
+The name of the `table` provided in the request to match.
+
+`match_field`::
+The fields in the input to match against the table.
+
+*Description*
+
+`LOOKUP` matches values from the input against a `table` provided in the request,
+adding the other fields from the `table` to the output.
+
+*Examples*
+
+// tag::examples[]
+[source,console]
+----
+POST /_query?format=txt
+{
+  "query": """
+      FROM library
+    | SORT page_count DESC
+    | KEEP name, author
+    | LOOKUP era ON author
+    | LIMIT 5
+  """,
+  "tables": {
+    "era": {
+      "author:keyword": ["Frank Herbert", "Peter F. Hamilton", "Vernor Vinge", "Alastair Reynolds", "James S.A. Corey"],
+      "era:keyword"   : [ "The New Wave",           "Diamond",      "Diamond",           "Diamond",           "Hadron"]
+    }
+  }
+}
+----
+// TEST[setup:library]
+
+Which returns:
+
+[source,text]
+----
+        name        |     author      |      era
+--------------------+-----------------+---------------
+Pandora's Star      |Peter F. Hamilton|Diamond
+A Fire Upon the Deep|Vernor Vinge     |Diamond
+Dune                |Frank Herbert    |The New Wave
+Revelation Space    |Alastair Reynolds|Diamond
+Leviathan Wakes     |James S.A. Corey |Hadron
+----
+// TESTRESPONSE[s/\|/\\|/ s/\+/\\+/]
+// TESTRESPONSE[non_json]
+// end::examples[]
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
index 587b503d1f8f5..a42f02a22aba8 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.test.rest.TestFeatureService;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.CsvTestUtils;
+import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.core.CsvSpecReader.CsvTestCase;
 import org.elasticsearch.xpack.esql.core.SpecReader;
 import org.elasticsearch.xpack.esql.plugin.EsqlFeatures;
@@ -38,11 +39,15 @@
 import java.io.IOException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.TreeMap;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
 import java.util.stream.Stream;
 
 import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
@@ -192,11 +197,16 @@ protected static void checkCapabilities(RestClient client, TestFeatureService te
     protected final void doTest() throws Throwable {
         RequestObjectBuilder builder = new RequestObjectBuilder(randomFrom(XContentType.values()));
 
+        if (testCase.query.toUpperCase(Locale.ROOT).contains("LOOKUP")) {
+            builder.tables(tables());
+        }
+
         Map<String, Object> answer = runEsql(
             builder.query(testCase.query),
             testCase.expectedWarnings(false),
             testCase.expectedWarningsRegex()
         );
+
         var expectedColumnsWithValues = loadCsvSpecValues(testCase.expectedResults);
 
         var metadata = answer.get("columns");
@@ -306,4 +316,50 @@ public static void assertRequestBreakerEmpty() throws Exception {
             }
         });
     }
+
+    /**
+     * "tables" parameter sent if there is a LOOKUP in the request. If you
+     * add to this, you must also add to {@link EsqlTestUtils#tables};
+     */
+    private Map<String, Map<String, List<?>>> tables() {
+        Map<String, Map<String, List<?>>> tables = new TreeMap<>();
+        tables.put(
+            "int_number_names",
+            EsqlTestUtils.table(
+                Map.entry("int:integer", IntStream.range(0, 10).boxed().toList()),
+                Map.entry("name:keyword", IntStream.range(0, 10).mapToObj(EsqlTestUtils::numberName).toList())
+            )
+        );
+        tables.put(
+            "long_number_names",
+            EsqlTestUtils.table(
+                Map.entry("long:long", LongStream.range(0, 10).boxed().toList()),
+                Map.entry("name:keyword", IntStream.range(0, 10).mapToObj(EsqlTestUtils::numberName).toList())
+            )
+        );
+        tables.put(
+            "double_number_names",
+            EsqlTestUtils.table(
+                Map.entry("double:double", List.of(2.03, 2.08)),
+                Map.entry("name:keyword", List.of("two point zero three", "two point zero eight"))
+            )
+        );
+        tables.put(
+            "double_number_names_with_null",
+            EsqlTestUtils.table(
+                Map.entry("double:double", List.of(2.03, 2.08, 0.0)),
+                Map.entry("name:keyword", Arrays.asList("two point zero three", "two point zero eight", null))
+            )
+        );
+        tables.put(
+            "big",
+            EsqlTestUtils.table(
+                Map.entry("aa:keyword", List.of("foo", "bar", "baz", "foo")),
+                Map.entry("ab:keyword", List.of("zoo", "zop", "zoi", "foo")),
+                Map.entry("na:integer", List.of(1, 10, 100, 2)),
+                Map.entry("nb:integer", List.of(-1, -10, -100, -2))
+            )
+        );
+        return tables;
+    }
 }
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
index 3f21c9da31861..349f9445030c4 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java
@@ -113,6 +113,8 @@ public static class RequestObjectBuilder {
         private final XContentBuilder builder;
         private boolean isBuilt = false;
 
+        private Map<String, Map<String, List<?>>> tables;
+
         private Boolean keepOnCompletion = null;
 
         public RequestObjectBuilder() throws IOException {
@@ -129,6 +131,11 @@ public RequestObjectBuilder query(String query) throws IOException {
             return this;
         }
 
+        public RequestObjectBuilder tables(Map<String, Map<String, List<?>>> tables) {
+            this.tables = tables;
+            return this;
+        }
+
         public RequestObjectBuilder columnar(boolean columnar) throws IOException {
             builder.field("columnar", columnar);
             return this;
@@ -173,6 +180,9 @@ public RequestObjectBuilder pragmas(Settings pragmas) throws IOException {
 
         public RequestObjectBuilder build() throws IOException {
             if (isBuilt == false) {
+                if (tables != null) {
+                    builder.field("tables", tables);
+                }
                 builder.endObject();
                 isBuilt = true;
             }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
index 86b931046e6db..7096c12022729 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BlockUtils;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
@@ -44,12 +45,13 @@
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.regex.Pattern;
 
 import static java.util.Collections.emptyList;
@@ -268,9 +270,13 @@ public static void assertWarnings(List<String> warnings, List<String> allowedWar
         }
     }
 
-    static Map<String, Map<String, Column>> tables() {
+    /**
+     * "tables" provided in the context for the LOOKUP command. If you
+     * add to this, you must also add to {@code EsqlSpecTestCase#tables};
+     */
+    public static Map<String, Map<String, Column>> tables() {
         BlockFactory factory = new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE);
-        Map<String, Map<String, Column>> tables = new HashMap<>();
+        Map<String, Map<String, Column>> tables = new TreeMap<>();
         try (
             IntBlock.Builder ints = factory.newIntBlockBuilder(10);
             LongBlock.Builder longs = factory.newLongBlockBuilder(10);
@@ -279,19 +285,7 @@ static Map<String, Map<String, Column>> tables() {
             for (int i = 0; i < 10; i++) {
                 ints.appendInt(i);
                 longs.appendLong(i);
-                names.appendBytesRef(new BytesRef(switch (i) {
-                    case 0 -> "zero";
-                    case 1 -> "one";
-                    case 2 -> "two";
-                    case 3 -> "three";
-                    case 4 -> "four";
-                    case 5 -> "five";
-                    case 6 -> "six";
-                    case 7 -> "seven";
-                    case 8 -> "eight";
-                    case 9 -> "nine";
-                    default -> throw new IllegalArgumentException();
-                }));
+                names.appendBytesRef(new BytesRef(numberName(i)));
             }
 
             IntBlock intsBlock = ints.build();
@@ -299,13 +293,104 @@ static Map<String, Map<String, Column>> tables() {
             BytesRefBlock namesBlock = names.build();
             tables.put(
                 "int_number_names",
-                Map.of("int", new Column(DataType.INTEGER, intsBlock), "name", new Column(DataType.KEYWORD, namesBlock))
+                table(
+                    Map.entry("int", new Column(DataType.INTEGER, intsBlock)),
+                    Map.entry("name", new Column(DataType.KEYWORD, namesBlock))
+                )
             );
             tables.put(
                 "long_number_names",
-                Map.of("long", new Column(DataType.LONG, longsBlock), "name", new Column(DataType.KEYWORD, namesBlock))
+                table(Map.entry("long", new Column(DataType.LONG, longsBlock)), Map.entry("name", new Column(DataType.KEYWORD, namesBlock)))
+            );
+        }
+        for (boolean hasNull : new boolean[] { true, false }) {
+            try (
+                DoubleBlock.Builder doubles = factory.newDoubleBlockBuilder(2);
+                BytesRefBlock.Builder names = factory.newBytesRefBlockBuilder(2);
+            ) {
+                doubles.appendDouble(2.03);
+                names.appendBytesRef(new BytesRef("two point zero three"));
+                doubles.appendDouble(2.08);
+                names.appendBytesRef(new BytesRef("two point zero eight"));
+                if (hasNull) {
+                    doubles.appendDouble(0.0);
+                    names.appendNull();
+                }
+                tables.put(
+                    "double_number_names" + (hasNull ? "_with_null" : ""),
+                    table(
+                        Map.entry("double", new Column(DataType.DOUBLE, doubles.build())),
+                        Map.entry("name", new Column(DataType.KEYWORD, names.build()))
+                    )
+                );
+            }
+        }
+        try (
+            BytesRefBlock.Builder aa = factory.newBytesRefBlockBuilder(3);
+            BytesRefBlock.Builder ab = factory.newBytesRefBlockBuilder(3);
+            IntBlock.Builder na = factory.newIntBlockBuilder(3);
+            IntBlock.Builder nb = factory.newIntBlockBuilder(3);
+        ) {
+            aa.appendBytesRef(new BytesRef("foo"));
+            ab.appendBytesRef(new BytesRef("zoo"));
+            na.appendInt(1);
+            nb.appendInt(-1);
+
+            aa.appendBytesRef(new BytesRef("bar"));
+            ab.appendBytesRef(new BytesRef("zop"));
+            na.appendInt(10);
+            nb.appendInt(-10);
+
+            aa.appendBytesRef(new BytesRef("baz"));
+            ab.appendBytesRef(new BytesRef("zoi"));
+            na.appendInt(100);
+            nb.appendInt(-100);
+
+            aa.appendBytesRef(new BytesRef("foo"));
+            ab.appendBytesRef(new BytesRef("foo"));
+            na.appendInt(2);
+            nb.appendInt(-2);
+
+            tables.put(
+                "big",
+                table(
+                    Map.entry("aa", new Column(DataType.KEYWORD, aa.build())),
+                    Map.entry("ab", new Column(DataType.KEYWORD, ab.build())),
+                    Map.entry("na", new Column(DataType.INTEGER, na.build())),
+                    Map.entry("nb", new Column(DataType.INTEGER, nb.build()))
+                )
             );
         }
+
         return unmodifiableMap(tables);
     }
+
+    /**
+     * Builds a table from the provided parameters. This isn't just a call to
+     * {@link Map#of} because we want to maintain sort order of the columns
+     */
+    @SafeVarargs
+    public static <T> Map<String, T> table(Map.Entry<String, T>... kv) {
+        Map<String, T> table = new LinkedHashMap<>();
+        for (Map.Entry<String, T> stringTEntry : kv) {
+            table.put(stringTEntry.getKey(), stringTEntry.getValue());
+        }
+        return table;
+    }
+
+    public static String numberName(int i) {
+        return switch (i) {
+            case 0 -> "zero";
+            case 1 -> "one";
+            case 2 -> "two";
+            case 3 -> "three";
+            case 4 -> "four";
+            case 5 -> "five";
+            case 6 -> "six";
+            case 7 -> "seven";
+            case 8 -> "eight";
+            case 9 -> "nine";
+            default -> throw new IllegalArgumentException();
+        };
+    }
 }
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec
index f6dc28fa5db12..377d6d6678032 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec
@@ -1,11 +1,259 @@
+keywordByInt
+required_capability: lookup_command
+FROM employees
+| SORT emp_no
+| LIMIT 4
+| RENAME languages AS int
+| LOOKUP int_number_names ON int
+| RENAME int AS languages, name AS lang_name
+| KEEP emp_no, languages, lang_name
+;
+
+emp_no:integer | languages:integer | lang_name:keyword
+         10001 |                 2 | two
+         10002 |                 5 | five
+         10003 |                 4 | four
+         10004 |                 5 | five
+;
+
+keywordByMvInt
+required_capability: lookup_command
+ROW int=[1, 2, 3]
+| LOOKUP int_number_names ON int
+;
+
+int:integer | name:keyword
+  [1, 2, 3] | [one, two, three]
+;
+
+keywordByDupeInt
+required_capability: lookup_command
+ROW int=[1, 1, 1]
+| LOOKUP int_number_names ON int
+;
+
+int:integer | name:keyword
+  [1, 1, 1] | [one, one, one]
+;
+
+intByKeyword
+required_capability: lookup_command
+ROW name="two"
+| LOOKUP int_number_names ON name
+;
+
+name:keyword | int:integer 
+         two | 2
+;
+
+
+keywordByLong
+required_capability: lookup_command
+FROM employees
+| SORT emp_no
+| LIMIT 4
+| RENAME languages.long AS long
+| LOOKUP long_number_names ON long
+| RENAME long AS languages, name AS lang_name
+| KEEP emp_no, languages, lang_name
+;
+
+emp_no:integer | languages:long | lang_name:keyword
+         10001 |              2 | two
+         10002 |              5 | five
+         10003 |              4 | four
+         10004 |              5 | five
+;
+
+longByKeyword
+required_capability: lookup_command
+ROW name="two"
+| LOOKUP long_number_names ON name
+;
+
+name:keyword | long:long
+         two |         2
+;
+
+keywordByFloat
+required_capability: lookup_command
+FROM employees
+| SORT emp_no
+| LIMIT 4
+| RENAME height AS double
+| LOOKUP double_number_names ON double
+| RENAME double AS height, name AS height_name
+| KEEP emp_no, height, height_name
+;
+
+emp_no:integer | height:double | height_name:keyword
+         10001 |          2.03 | two point zero three
+         10002 |          2.08 | two point zero eight
+         10003 |          1.83 | null
+         10004 |          1.78 | null
+;
+
+floatByKeyword
+required_capability: lookup_command
+ROW name="two point zero eight"
+| LOOKUP double_number_names ON name
+;
+
+    name:keyword     | double:double
+two point zero eight |          2.08
+;
+
+floatByNullMissing
+required_capability: lookup_command
+ROW name=null
+| LOOKUP double_number_names ON name
+;
+
+name:null | double:double
+     null |          null
+;
 
+floatByNullMatching
+required_capability: lookup_command
+ROW name=null
+| LOOKUP double_number_names_with_null ON name
+;
+
+name:null | double:double
+     null |             0
+;
+
+intIntByKeywordKeyword
+required_capability: lookup_command
+ROW aa="foo", ab="zoo"
+| LOOKUP big ON aa, ab
+;
+
+aa:keyword | ab:keyword | na:integer | nb:integer
+foo        | zoo        |          1 |         -1
+;
+
+intIntByKeywordKeywordMissing
+required_capability: lookup_command
+ROW aa="foo", ab="zoi"
+| LOOKUP big ON aa, ab
+;
+
+aa:keyword | ab:keyword | na:integer | nb:integer
+foo        | zoi        |       null |       null
+;
+
+intIntByKeywordKeywordSameValues
+required_capability: lookup_command
+ROW aa="foo", ab="foo"
+| LOOKUP big ON aa, ab
+;
+
+aa:keyword | ab:keyword | na:integer | nb:integer
+foo        | foo        |          2 |         -2
+;
+
+intIntByKeywordKeywordSameValuesMissing
+required_capability: lookup_command
+ROW aa="bar", ab="bar"
+| LOOKUP big ON aa, ab
+;
+
+aa:keyword | ab:keyword | na:integer | nb:integer
+bar        | bar        |       null |       null
+;
+
+lookupBeforeStats
+required_capability: lookup_command
+  FROM employees
+| RENAME languages AS int
+| LOOKUP int_number_names ON int
+| RENAME name AS languages
+| STATS height=ROUND(AVG(height), 3) BY languages
+| SORT height ASC;
+
+height:double | languages:keyword
+        1.694 | four
+        1.732 | one
+        1.762 | two
+        1.764 | three
+        1.809 | null        
+        1.847 | five
+;
+
+lookupAfterStats
+required_capability: lookup_command
+  FROM employees
+| STATS int=TO_INT(AVG(height))
+| LOOKUP int_number_names ON int
+| KEEP name;
+
+name:keyword
+two
+;
+
+// Makes sure the LOOKUP squashes previous names 
+doesNotDuplicateNames
+required_capability: lookup_command
+FROM employees
+| SORT emp_no
+| LIMIT 4
+| RENAME languages.long AS long
+| EVAL name = CONCAT(first_name, " ", last_name)
+| LOOKUP long_number_names ON long
+| RENAME long AS languages
+| KEEP emp_no, languages, name
+;
 
+emp_no:integer | languages:long | name:keyword
+         10001 |              2 | two
+         10002 |              5 | five
+         10003 |              4 | four
+         10004 |              5 | five
+;
+
+lookupBeforeSort
+required_capability: lookup_command
+FROM employees
+| WHERE emp_no < 10005
+| RENAME languages AS int
+| LOOKUP int_number_names ON int
+| RENAME name AS languages
+| KEEP languages, emp_no
+| SORT languages ASC, emp_no ASC
+;
+
+languages:keyword | emp_no:integer
+             five | 10002
+             five | 10004
+             four | 10003
+              two | 10001
+;
+
+lookupAfterSort
+required_capability: lookup_command
+FROM employees
+| WHERE emp_no < 10005
+| SORT languages ASC, emp_no ASC
+| RENAME languages AS int
+| LOOKUP int_number_names ON int
+| RENAME name AS languages
+| KEEP languages, emp_no
+;
+
+languages:keyword | emp_no:integer
+              two | 10001
+             four | 10003
+             five | 10002
+             five | 10004
+;
 
 //
 // Make sure that the new LOOKUP syntax doesn't clash with any existing things
 // named "lookup"
 //
 rowNamedLookup
+required_capability: lookup_command
 ROW lookup = "a"
 ;
 
@@ -14,6 +262,7 @@ lookup:keyword
 ;
 
 rowNamedLOOKUP
+required_capability: lookup_command
 ROW LOOKUP = "a"
 ;
 
@@ -22,6 +271,7 @@ LOOKUP:keyword
 ;
 
 evalNamedLookup
+required_capability: lookup_command
 ROW a = "a" | EVAL lookup = CONCAT(a, "1")
 ;
 
@@ -30,6 +280,7 @@ a:keyword | lookup:keyword
 ;
 
 dissectNamedLookup
+required_capability: lookup_command
 row a = "foo bar" | dissect a "foo %{lookup}";
 
 a:keyword | lookup:keyword
@@ -37,6 +288,7 @@ a:keyword | lookup:keyword
 ;
 
 renameIntoLookup
+required_capability: lookup_command
 row a = "foo bar" | RENAME a AS lookup;
 
 lookup:keyword
@@ -44,6 +296,7 @@ lookup:keyword
 ;
 
 sortOnLookup
+required_capability: lookup_command
 ROW lookup = "a" | SORT lookup
 ;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/Column.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/Column.java
index 1495c90dc67bf..a19dafba1559b 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/Column.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/Column.java
@@ -14,14 +14,19 @@
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.planner.PlannerUtils;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 
 import java.io.IOException;
 
 /**
- * A column of data provided in the request.
+ * A "column" from a {@code table} provided in the request.
  */
 public record Column(DataType type, Block values) implements Releasable, Writeable {
+    public Column {
+        assert PlannerUtils.toElementType(type) == values.elementType();
+    }
+
     public Column(BlockStreamInput in) throws IOException {
         this(EsqlDataTypes.fromTypeName(in.readString()), in.readNamedWriteable(Block.class));
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index 13ad47850cd01..75600e64e3e90 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -53,9 +53,9 @@ public class EsqlCapabilities {
     private static final String METADATA_IGNORED_FIELD = "metadata_field_ignored";
 
     /**
-     * Support for requesting the "LOOKUP" command.
+     * Support for the "LOOKUP" command.
      */
-    private static final String LOOKUP = "lookup";
+    private static final String LOOKUP_COMMAND = "lookup_command";
 
     /**
      * Support for requesting the "REPEAT" command.
@@ -80,7 +80,7 @@ private static Set<String> capabilities() {
         caps.add(REPEAT);
 
         if (Build.current().isSnapshot()) {
-            caps.add(LOOKUP);
+            caps.add(LOOKUP_COMMAND);
         }
 
         /*
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ParseTables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ParseTables.java
index c9748ee214820..97728196a0ced 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ParseTables.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ParseTables.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.core.Releasables;
@@ -23,9 +24,9 @@
 import org.elasticsearch.xpack.esql.core.type.DataType;
 
 import java.io.IOException;
+import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Set;
-import java.util.TreeMap;
 
 /**
  * Parses the {@code tables} request body parameter.
@@ -69,7 +70,7 @@ void parseTables() throws IOException {
      * so we can be sure we'll always have a type.
      */
     private Map<String, Column> parseTable() throws IOException {
-        Map<String, Column> columns = new TreeMap<>();
+        Map<String, Column> columns = new LinkedHashMap<>();
         boolean success = false;
         try {
             if (p.currentToken() != XContentParser.Token.START_OBJECT) {
@@ -112,6 +113,7 @@ private Column parseColumn(String type) throws IOException {
             case "integer" -> parseIntColumn();
             case "keyword" -> parseKeywordColumn();
             case "long" -> parseLongColumn();
+            case "double" -> parseDoubleColumn();
             default -> throw new XContentParseException(p.getTokenLocation(), "unsupported type [" + type + "]");
         };
     }
@@ -245,4 +247,46 @@ private void appendLong(LongBlock.Builder builder) throws IOException {
         }
         builder.appendLong(p.longValue());
     }
+
+    private Column parseDoubleColumn() throws IOException {
+        try (DoubleBlock.Builder builder = blockFactory.newDoubleBlockBuilder(100)) { // TODO 100?!
+            XContentParser.Token token = p.nextToken();
+            if (token != XContentParser.Token.START_ARRAY) {
+                throw new XContentParseException(p.getTokenLocation(), "expected " + XContentParser.Token.START_ARRAY);
+            }
+            while (true) {
+                switch (p.nextToken()) {
+                    case END_ARRAY -> {
+                        return new Column(DataType.DOUBLE, builder.build());
+                    }
+                    case START_ARRAY -> parseDoubleArray(builder);
+                    case VALUE_NULL -> builder.appendNull();
+                    case VALUE_NUMBER, VALUE_STRING -> appendDouble(builder);
+                    default -> throw new XContentParseException(p.getTokenLocation(), "expected number, array of numbers, or null");
+                }
+            }
+        }
+    }
+
+    private void parseDoubleArray(DoubleBlock.Builder builder) throws IOException {
+        builder.beginPositionEntry();
+        while (true) {
+            switch (p.nextToken()) {
+                case END_ARRAY -> {
+                    builder.endPositionEntry();
+                    return;
+                }
+                case VALUE_NUMBER, VALUE_STRING -> appendDouble(builder);
+                default -> throw new XContentParseException(p.getTokenLocation(), "expected number");
+            }
+        }
+    }
+
+    private void appendDouble(DoubleBlock.Builder builder) throws IOException {
+        length += Double.BYTES;
+        if (length > MAX_LENGTH) {
+            throw new XContentParseException(p.getTokenLocation(), "tables too big");
+        }
+        builder.appendDouble(p.doubleValue());
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index c8d3207d016e4..70fbe17a7d470 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -9,7 +9,9 @@
 
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
+import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
+import org.elasticsearch.xpack.esql.Column;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.core.analyzer.AnalyzerRules;
@@ -68,10 +70,13 @@
 import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Keep;
+import org.elasticsearch.xpack.esql.plan.logical.Lookup;
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.Rename;
 import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
 import org.elasticsearch.xpack.esql.stats.FeatureMetric;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
@@ -119,7 +124,14 @@ public class Analyzer extends ParameterizedRuleExecutor<LogicalPlan, AnalyzerCon
     private static final Iterable<RuleExecutor.Batch<LogicalPlan>> rules;
 
     static {
-        var init = new Batch<>("Initialize", Limiter.ONCE, new ResolveTable(), new ResolveEnrich(), new ResolveFunctions());
+        var init = new Batch<>(
+            "Initialize",
+            Limiter.ONCE,
+            new ResolveTable(),
+            new ResolveEnrich(),
+            new ResolveLookupTables(),
+            new ResolveFunctions()
+        );
         var resolution = new Batch<>("Resolution", new ResolveRefs(), new ImplicitCasting());
         var finish = new Batch<>("Finish Analysis", Limiter.ONCE, new AddImplicitLimit());
         rules = List.of(init, resolution, finish);
@@ -317,10 +329,60 @@ private static NamedExpression createEnrichFieldExpression(
         }
     }
 
-    private static class ResolveRefs extends BaseAnalyzerRule {
+    private static class ResolveLookupTables extends ParameterizedAnalyzerRule<Lookup, AnalyzerContext> {
 
+        @Override
+        protected LogicalPlan rule(Lookup lookup, AnalyzerContext context) {
+            // the parser passes the string wrapped in a literal
+            Source source = lookup.source();
+            Expression tableNameExpression = lookup.tableName();
+            String tableName = lookup.tableName().toString();
+            Map<String, Map<String, Column>> tables = context.configuration().tables();
+            LocalRelation localRelation = null;
+
+            if (tables.containsKey(tableName) == false) {
+                String message = "Unknown table [" + tableName + "]";
+                // typos check
+                List<String> potentialMatches = StringUtils.findSimilar(tableName, tables.keySet());
+                if (CollectionUtils.isEmpty(potentialMatches) == false) {
+                    message = UnresolvedAttribute.errorMessage(tableName, potentialMatches).replace("column", "table");
+                }
+                tableNameExpression = new UnresolvedAttribute(tableNameExpression.source(), tableName, null, message);
+            }
+            // wrap the table in a local relationship for idiomatic field resolution
+            else {
+                localRelation = tableMapAsRelation(source, tables.get(tableName));
+                // postpone the resolution for ResolveRefs
+            }
+
+            return new Lookup(source, lookup.child(), tableNameExpression, lookup.matchFields(), localRelation);
+        }
+
+        private LocalRelation tableMapAsRelation(Source source, Map<String, Column> mapTable) {
+            Block[] blocks = new Block[mapTable.size()];
+
+            List<Attribute> attributes = new ArrayList<>(blocks.length);
+            int i = 0;
+            for (Map.Entry<String, Column> entry : mapTable.entrySet()) {
+                String name = entry.getKey();
+                Column column = entry.getValue();
+                // create a fake ES field - alternative is to use a ReferenceAttribute
+                EsField field = new EsField(name, column.type(), Map.of(), false, false);
+                attributes.add(new FieldAttribute(source, null, name, field));
+                // prepare the block for the supplier
+                blocks[i++] = column.values();
+            }
+            LocalSupplier supplier = LocalSupplier.of(blocks);
+            return new LocalRelation(source, attributes, supplier);
+        }
+    }
+
+    private static class ResolveRefs extends BaseAnalyzerRule {
         @Override
         protected LogicalPlan doRule(LogicalPlan plan) {
+            if (plan.childrenResolved() == false) {
+                return plan;
+            }
             final List<Attribute> childrenOutput = new ArrayList<>();
 
             for (LogicalPlan child : plan.children()) {
@@ -356,6 +418,10 @@ protected LogicalPlan doRule(LogicalPlan plan) {
                 return resolveMvExpand(p, childrenOutput);
             }
 
+            if (plan instanceof Lookup l) {
+                return resolveLookup(l, childrenOutput);
+            }
+
             return plan.transformExpressionsOnly(UnresolvedAttribute.class, ua -> maybeResolveAttribute(ua, childrenOutput));
         }
 
@@ -438,6 +504,70 @@ private LogicalPlan resolveMvExpand(MvExpand p, List<Attribute> childrenOutput)
             return p;
         }
 
+        private LogicalPlan resolveLookup(Lookup l, List<Attribute> childrenOutput) {
+            // check if the table exists before performing any resolution
+            if (l.localRelation() == null) {
+                return l;
+            }
+
+            // check the on field against both the child output and the inner relation
+            List<NamedExpression> matchFields = new ArrayList<>(l.matchFields().size());
+            List<Attribute> localOutput = l.localRelation().output();
+            boolean modified = false;
+
+            for (NamedExpression ne : l.matchFields()) {
+                NamedExpression matchFieldChildReference = ne;
+                if (ne instanceof UnresolvedAttribute ua && ua.customMessage() == false) {
+                    modified = true;
+                    Attribute joinedAttribute = maybeResolveAttribute(ua, localOutput);
+                    // can't find the field inside the local relation
+                    if (joinedAttribute instanceof UnresolvedAttribute lua) {
+                        // adjust message
+                        matchFieldChildReference = lua.withUnresolvedMessage(
+                            lua.unresolvedMessage().replace("Unknown column", "Unknown column in lookup target")
+                        );
+                    } else {
+                        // check also the child output by resolving to it
+                        Attribute attr = maybeResolveAttribute(ua, childrenOutput);
+                        matchFieldChildReference = attr;
+                        if (attr instanceof UnresolvedAttribute == false) {
+                            /*
+                             * If they do, make sure the data types line up. If either is
+                             * null it's fine to match it against anything.
+                             */
+                            boolean dataTypesOk = joinedAttribute.dataType().equals(attr.dataType());
+                            if (false == dataTypesOk) {
+                                dataTypesOk = joinedAttribute.dataType() == DataType.NULL || attr.dataType() == DataType.NULL;
+                            }
+                            if (false == dataTypesOk) {
+                                dataTypesOk = joinedAttribute.dataType().equals(KEYWORD) && attr.dataType().equals(TEXT);
+                            }
+                            if (false == dataTypesOk) {
+                                matchFieldChildReference = new UnresolvedAttribute(
+                                    attr.source(),
+                                    attr.name(),
+                                    attr.qualifier(),
+                                    attr.id(),
+                                    "column type mismatch, table column was ["
+                                        + joinedAttribute.dataType().typeName()
+                                        + "] and original column was ["
+                                        + attr.dataType().typeName()
+                                        + "]",
+                                    null
+                                );
+                            }
+                        }
+                    }
+                }
+
+                matchFields.add(matchFieldChildReference);
+            }
+            if (modified) {
+                return new Lookup(l.source(), l.child(), l.tableName(), matchFields, l.localRelation());
+            }
+            return l;
+        }
+
         private Attribute maybeResolveAttribute(UnresolvedAttribute ua, List<Attribute> childrenOutput) {
             if (ua.customMessage()) {
                 return ua;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
index 87fcf05383128..367ba682274c9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
+import org.elasticsearch.xpack.esql.plan.logical.Lookup;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.RegexExtract;
 import org.elasticsearch.xpack.esql.plan.logical.Row;
@@ -124,7 +125,21 @@ else if (p.resolved()) {
                 var aggs = agg.aggregates();
                 int size = aggs.size() - groupings.size();
                 aggs.subList(0, size).forEach(unresolvedExpressions);
-            } else {
+            }
+            // similar approach for Lookup
+            else if (p instanceof Lookup lookup) {
+                // first check the table
+                var tableName = lookup.tableName();
+                if (tableName instanceof Unresolvable u) {
+                    failures.add(fail(tableName, u.unresolvedMessage()));
+                }
+                // only after that check the match fields
+                else {
+                    lookup.matchFields().forEach(unresolvedExpressions);
+                }
+            }
+
+            else {
                 p.forEachExpression(unresolvedExpressions);
             }
         });
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
index 624c9f5c65ca5..20d9907c61ba2 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java
@@ -179,10 +179,13 @@
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Grok;
+import org.elasticsearch.xpack.esql.plan.logical.Lookup;
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.TopN;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
 import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.DissectExec;
 import org.elasticsearch.xpack.esql.plan.physical.EnrichExec;
@@ -196,7 +199,9 @@
 import org.elasticsearch.xpack.esql.plan.physical.FilterExec;
 import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
 import org.elasticsearch.xpack.esql.plan.physical.GrokExec;
+import org.elasticsearch.xpack.esql.plan.physical.HashJoinExec;
 import org.elasticsearch.xpack.esql.plan.physical.LimitExec;
+import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.MvExpandExec;
 import org.elasticsearch.xpack.esql.plan.physical.OrderExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
@@ -277,6 +282,8 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(PhysicalPlan.class, FragmentExec.class, PlanNamedTypes::writeFragmentExec, PlanNamedTypes::readFragmentExec),
             of(PhysicalPlan.class, GrokExec.class, PlanNamedTypes::writeGrokExec, PlanNamedTypes::readGrokExec),
             of(PhysicalPlan.class, LimitExec.class, PlanNamedTypes::writeLimitExec, PlanNamedTypes::readLimitExec),
+            of(PhysicalPlan.class, LocalSourceExec.class, (out, v) -> v.writeTo(out), LocalSourceExec::new),
+            of(PhysicalPlan.class, HashJoinExec.class, (out, v) -> v.writeTo(out), HashJoinExec::new),
             of(PhysicalPlan.class, MvExpandExec.class, PlanNamedTypes::writeMvExpandExec, PlanNamedTypes::readMvExpandExec),
             of(PhysicalPlan.class, OrderExec.class, PlanNamedTypes::writeOrderExec, PlanNamedTypes::readOrderExec),
             of(PhysicalPlan.class, ProjectExec.class, PlanNamedTypes::writeProjectExec, PlanNamedTypes::readProjectExec),
@@ -292,7 +299,10 @@ public static List<PlanNameRegistry.Entry> namedTypeEntries() {
             of(LogicalPlan.class, EsqlProject.class, PlanNamedTypes::writeEsqlProject, PlanNamedTypes::readEsqlProject),
             of(LogicalPlan.class, Filter.class, PlanNamedTypes::writeFilter, PlanNamedTypes::readFilter),
             of(LogicalPlan.class, Grok.class, PlanNamedTypes::writeGrok, PlanNamedTypes::readGrok),
+            of(LogicalPlan.class, Join.class, (out, p) -> p.writeTo(out), Join::new),
             of(LogicalPlan.class, Limit.class, PlanNamedTypes::writeLimit, PlanNamedTypes::readLimit),
+            of(LogicalPlan.class, LocalRelation.class, (out, p) -> p.writeTo(out), LocalRelation::new),
+            of(LogicalPlan.class, Lookup.class, (out, p) -> p.writeTo(out), Lookup::new),
             of(LogicalPlan.class, MvExpand.class, PlanNamedTypes::writeMvExpand, PlanNamedTypes::readMvExpand),
             of(LogicalPlan.class, OrderBy.class, PlanNamedTypes::writeOrderBy, PlanNamedTypes::readOrderBy),
             of(LogicalPlan.class, Project.class, PlanNamedTypes::writeProject, PlanNamedTypes::readProject),
@@ -1072,7 +1082,7 @@ static void writeTopN(PlanStreamOutput out, TopN topN) throws IOException {
 
     // -- BinaryComparison
 
-    static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) throws IOException {
+    public static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) throws IOException {
         var source = Source.readFrom(in);
         EsqlBinaryComparison.BinaryComparisonOperation operation = EsqlBinaryComparison.BinaryComparisonOperation.readFromStream(in);
         var left = in.readExpression();
@@ -1082,7 +1092,7 @@ static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) t
         return operation.buildNewInstance(source, left, right);
     }
 
-    static void writeBinComparison(PlanStreamOutput out, EsqlBinaryComparison binaryComparison) throws IOException {
+    public static void writeBinComparison(PlanStreamOutput out, EsqlBinaryComparison binaryComparison) throws IOException {
         binaryComparison.source().writeTo(out);
         binaryComparison.getFunctionType().writeTo(out);
         out.writeExpression(binaryComparison.left());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
index f7380588fbd77..45662d13e2618 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.session.EsqlConfiguration;
 
@@ -75,7 +76,7 @@ public PlanStreamOutput(
     }
 
     public void writeLogicalPlanNode(LogicalPlan logicalPlan) throws IOException {
-        assert logicalPlan.children().size() <= 1;
+        assert logicalPlan.children().size() <= 1 || (logicalPlan instanceof Join && logicalPlan.children().size() == 2);
         writeNamed(LogicalPlan.class, logicalPlan);
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
index 66bd69529f34e..c5f96988a2ed5 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.TopN;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.planner.AbstractPhysicalOperationProviders;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
@@ -123,7 +124,7 @@ public LogicalPlan apply(LogicalPlan plan, LocalLogicalOptimizerContext localLog
         }
 
         private LogicalPlan missingToNull(LogicalPlan plan, SearchStats stats) {
-            if (plan instanceof EsRelation) {
+            if (plan instanceof EsRelation || plan instanceof LocalRelation) {
                 return plan;
             }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
index b5a288408809f..60b758e948518 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
@@ -68,10 +68,13 @@
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
+import org.elasticsearch.xpack.esql.plan.logical.Lookup;
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.RegexExtract;
 import org.elasticsearch.xpack.esql.plan.logical.TopN;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinType;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
 import org.elasticsearch.xpack.esql.planner.PlannerUtils;
@@ -121,6 +124,7 @@ protected static Batch<LogicalPlan> substitutions() {
         return new Batch<>(
             "Substitutions",
             Limiter.ONCE,
+            new ReplaceLookupWithJoin(),
             new RemoveStatsOverride(),
             // first extract nested expressions inside aggs
             new ReplaceStatsNestedExpressionWithEval(),
@@ -658,6 +662,11 @@ protected LogicalPlan rule(Limit limit) {
                         }
                     }
                 }
+            } else if (limit.child() instanceof Join join) {
+                if (join.config().type() == JoinType.LEFT && join.right() instanceof LocalRelation) {
+                    // This is a hash join from something like a lookup.
+                    return join.replaceChildren(limit.replaceChild(join.left()), join.right());
+                }
             }
             return limit;
         }
@@ -1261,6 +1270,19 @@ protected LogicalPlan rule(Limit plan) {
         }
     }
 
+    private static class ReplaceLookupWithJoin extends OptimizerRules.OptimizerRule<Lookup> {
+
+        ReplaceLookupWithJoin() {
+            super(TransformDirection.UP);
+        }
+
+        @Override
+        protected LogicalPlan rule(Lookup lookup) {
+            // left join between the main relation and the local, lookup relation
+            return new Join(lookup.source(), lookup.child(), lookup.localRelation(), lookup.joinConfig());
+        }
+    }
+
     /**
      * This adds an explicit TopN node to a plan that only has an OrderBy right before Lucene.
      * To date, the only known use case that "needs" this is a query of the form
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java
index 17ded5cf44c09..a0a3874a2c2de 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizer.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.esql.core.rule.RuleExecutor;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.Holder;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
@@ -28,6 +29,7 @@
 import org.elasticsearch.xpack.esql.plan.physical.EnrichExec;
 import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec;
 import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
+import org.elasticsearch.xpack.esql.plan.physical.HashJoinExec;
 import org.elasticsearch.xpack.esql.plan.physical.MvExpandExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.esql.plan.physical.ProjectExec;
@@ -117,6 +119,12 @@ public PhysicalPlan apply(PhysicalPlan plan) {
                     if (p instanceof MvExpandExec mvee) {
                         attributes.remove(mvee.expanded());
                     }
+                    if (p instanceof HashJoinExec join) {
+                        attributes.removeAll(join.addedFields());
+                        for (Equals cond : join.conditions()) {
+                            attributes.remove(cond.right());
+                        }
+                    }
                     if (p instanceof EnrichExec ee) {
                         for (NamedExpression enrichField : ee.enrichFields()) {
                             // TODO: why is this different then the remove above?
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
index b80edbe895ced..ec238c1fcf316 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java
@@ -51,6 +51,7 @@
 import org.elasticsearch.xpack.esql.plan.logical.Grok;
 import org.elasticsearch.xpack.esql.plan.logical.InlineStats;
 import org.elasticsearch.xpack.esql.plan.logical.Keep;
+import org.elasticsearch.xpack.esql.plan.logical.Lookup;
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
 import org.elasticsearch.xpack.esql.plan.logical.Rename;
 import org.elasticsearch.xpack.esql.plan.logical.Row;
@@ -445,7 +446,7 @@ public PlanFactory visitLookupCommand(EsqlBaseParser.LookupCommandContext ctx) {
 
         Literal tableName = new Literal(source, ctx.tableName.getText(), DataType.KEYWORD);
 
-        throw new ParsingException(source, "LOOKUP not yet supported");
+        return p -> new Lookup(source, p, tableName, matchFields, null /* localRelation will be resolved later*/);
     }
 
     interface PlanFactory extends Function<LogicalPlan, LogicalPlan> {}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java
new file mode 100644
index 0000000000000..690e4595f64e5
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinType;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Looks up values from the associated {@code tables}.
+ * The class is supposed to be substituted by a {@link Join}.
+ */
+public class Lookup extends UnaryPlan {
+    private final Expression tableName;
+    /**
+     * References to the input fields to match against the {@link #localRelation}.
+     */
+    private final List<NamedExpression> matchFields;
+    // initialized during the analysis phase for output and validation
+    // afterward, it is converted into a Join (BinaryPlan) hence why here it is not a child
+    private final LocalRelation localRelation;
+    private List<Attribute> lazyOutput;
+
+    public Lookup(
+        Source source,
+        LogicalPlan child,
+        Expression tableName,
+        List<NamedExpression> matchFields,
+        @Nullable LocalRelation localRelation
+    ) {
+        super(source, child);
+        this.tableName = tableName;
+        this.matchFields = matchFields;
+        this.localRelation = localRelation;
+    }
+
+    public Lookup(PlanStreamInput in) throws IOException {
+        super(Source.readFrom(in), in.readLogicalPlanNode());
+        this.tableName = in.readExpression();
+        this.matchFields = in.readNamedWriteableCollectionAsList(NamedExpression.class);
+        this.localRelation = in.readBoolean() ? new LocalRelation(in) : null;
+    }
+
+    public void writeTo(PlanStreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeLogicalPlanNode(child());
+        out.writeExpression(tableName);
+        out.writeNamedWriteableCollection(matchFields);
+        if (localRelation == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            localRelation.writeTo(out);
+        }
+    }
+
+    public Expression tableName() {
+        return tableName;
+    }
+
+    public List<NamedExpression> matchFields() {
+        return matchFields;
+    }
+
+    public LocalRelation localRelation() {
+        return localRelation;
+    }
+
+    public JoinConfig joinConfig() {
+        List<Expression> conditions = new ArrayList<>(matchFields.size());
+        List<Attribute> rhsOutput = Join.makeReference(localRelation.output());
+        for (NamedExpression lhs : matchFields) {
+            for (Attribute rhs : rhsOutput) {
+                if (lhs.name().equals(rhs.name())) {
+                    conditions.add(new Equals(source(), lhs, rhs));
+                    break;
+                }
+            }
+        }
+        return new JoinConfig(JoinType.LEFT, matchFields, conditions);
+    }
+
+    @Override
+    public boolean expressionsResolved() {
+        return tableName.resolved() && Resolvables.resolved(matchFields) && localRelation != null;
+    }
+
+    @Override
+    public UnaryPlan replaceChild(LogicalPlan newChild) {
+        return new Lookup(source(), newChild, tableName, matchFields, localRelation);
+    }
+
+    @Override
+    protected NodeInfo<? extends LogicalPlan> info() {
+        return NodeInfo.create(this, Lookup::new, child(), tableName, matchFields, localRelation);
+    }
+
+    @Override
+    public List<Attribute> output() {
+        if (lazyOutput == null) {
+            List<Attribute> rightSide = localRelation != null
+                ? Join.makeNullable(Join.makeReference(localRelation.output()))
+                : Expressions.asAttributes(matchFields);
+            lazyOutput = Join.mergeOutput(child().output(), rightSide, matchFields);
+        }
+        return lazyOutput;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        if (super.equals(o) == false) {
+            return false;
+        }
+        Lookup lookup = (Lookup) o;
+        return Objects.equals(tableName, lookup.tableName)
+            && Objects.equals(matchFields, lookup.matchFields)
+            && Objects.equals(localRelation, lookup.localRelation);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), tableName, matchFields, localRelation);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java
new file mode 100644
index 0000000000000..81ec67a28bbfd
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical.join;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
+import org.elasticsearch.xpack.esql.core.plan.logical.BinaryPlan;
+import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class Join extends BinaryPlan {
+
+    private final JoinConfig config;
+    // TODO: The matching attributes from the left and right logical plans should become part of the `expressions()`
+    // so that `references()` returns the attributes we actually rely on.
+    private List<Attribute> lazyOutput;
+
+    public Join(Source source, LogicalPlan left, LogicalPlan right, JoinConfig config) {
+        super(source, left, right);
+        this.config = config;
+    }
+
+    public Join(PlanStreamInput in) throws IOException {
+        super(Source.readFrom(in), in.readLogicalPlanNode(), in.readLogicalPlanNode());
+        this.config = new JoinConfig(in);
+    }
+
+    public void writeTo(PlanStreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeLogicalPlanNode(left());
+        out.writeLogicalPlanNode(right());
+        config.writeTo(out);
+    }
+
+    public JoinConfig config() {
+        return config;
+    }
+
+    @Override
+    protected NodeInfo<Join> info() {
+        return NodeInfo.create(this, Join::new, left(), right(), config);
+    }
+
+    @Override
+    public Join replaceChildren(List<LogicalPlan> newChildren) {
+        return new Join(source(), newChildren.get(0), newChildren.get(1), config);
+    }
+
+    public Join replaceChildren(LogicalPlan left, LogicalPlan right) {
+        return new Join(source(), left, right, config);
+    }
+
+    @Override
+    public List<Attribute> output() {
+        if (lazyOutput == null) {
+            lazyOutput = computeOutput();
+        }
+        return lazyOutput;
+    }
+
+    private List<Attribute> computeOutput() {
+        List<Attribute> right = makeReference(right().output());
+        return switch (config.type()) {
+            case LEFT -> // right side becomes nullable
+                mergeOutput(left().output(), makeNullable(right), config.matchFields());
+            case RIGHT -> // left side becomes nullable
+                mergeOutput(makeNullable(left().output()), right, config.matchFields());
+            case FULL -> // both sides become nullable
+                mergeOutput(makeNullable(left().output()), makeNullable(right), config.matchFields());
+            default -> // neither side becomes nullable
+                mergeOutput(left().output(), right, config.matchFields());
+        };
+    }
+
+    /**
+     * Merge output fields, left hand side wins in name conflicts <strong>except</strong>
+     * for fields defined in {@link JoinConfig#matchFields()}.
+     */
+    public static List<Attribute> mergeOutput(
+        List<? extends Attribute> lhs,
+        List<? extends Attribute> rhs,
+        List<NamedExpression> matchFields
+    ) {
+        List<Attribute> results = new ArrayList<>(lhs.size() + rhs.size());
+
+        for (Attribute a : lhs) {
+            if (rhs.contains(a) == false || matchFields.stream().anyMatch(m -> m.name().equals(a.name()))) {
+                results.add(a);
+            }
+        }
+        for (Attribute a : rhs) {
+            if (false == matchFields.stream().anyMatch(m -> m.name().equals(a.name()))) {
+                results.add(a);
+            }
+        }
+        return results;
+    }
+
+    /**
+     * Make fields references, so we don't check if they exist in the index.
+     * We do this for fields that we know don't come from the index.
+     * <p>
+     *   It's important that name is returned as a *reference* here
+     *   instead of a field. If it were a field we'd use SearchStats
+     *   on it and discover that it doesn't exist in the index. It doesn't!
+     *   We don't expect it to. It exists only in the lookup table.
+     *   TODO we should rework stats so we don't have to do this
+     * </p>
+     */
+    public static List<Attribute> makeReference(List<Attribute> output) {
+        List<Attribute> out = new ArrayList<>(output.size());
+        for (Attribute a : output) {
+            if (a.resolved()) {
+                out.add(new ReferenceAttribute(a.source(), a.name(), a.dataType(), a.qualifier(), a.nullable(), a.id(), a.synthetic()));
+            } else {
+                out.add(a);
+            }
+        }
+        return out;
+    }
+
+    public static List<Attribute> makeNullable(List<Attribute> output) {
+        List<Attribute> out = new ArrayList<>(output.size());
+        for (Attribute a : output) {
+            out.add(a.withNullability(Nullability.TRUE));
+        }
+        return out;
+    }
+
+    @Override
+    public boolean expressionsResolved() {
+        return config.expressionsResolved();
+    }
+
+    @Override
+    public boolean resolved() {
+        // resolve the join if
+        // - the children are resolved
+        // - the condition (if present) is resolved to a boolean
+        return childrenResolved() && expressionsResolved();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(config, left(), right());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        Join other = (Join) obj;
+        return config.equals(other.config) && Objects.equals(left(), other.left()) && Objects.equals(right(), other.right());
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java
new file mode 100644
index 0000000000000..b5cf5d9234c6b
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical.join;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Configuration for a {@code JOIN} style operation.
+ * @param matchFields fields that are merged from the left and right relations
+ * @param conditions when these conditions are true the rows are joined
+ */
+public record JoinConfig(JoinType type, List<NamedExpression> matchFields, List<Expression> conditions) implements Writeable {
+    public JoinConfig(StreamInput in) throws IOException {
+        this(
+            JoinType.readFrom(in),
+            in.readNamedWriteableCollectionAsList(NamedExpression.class),
+            in.readCollectionAsList(i -> ((PlanStreamInput) i).readExpression())
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        type.writeTo(out);
+        out.writeNamedWriteableCollection(matchFields);
+        out.writeCollection(conditions, (o, v) -> ((PlanStreamOutput) o).writeExpression(v));
+    }
+
+    public boolean expressionsResolved() {
+        return Resolvables.resolved(matchFields) && Resolvables.resolved(conditions);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinType.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinType.java
new file mode 100644
index 0000000000000..bd3ba43c25016
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinType.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.logical.join;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+
+public enum JoinType implements Writeable {
+    INNER(0, "INNER"),
+    LEFT(1, "LEFT OUTER"),
+    RIGHT(2, "RIGHT OUTER"),
+    FULL(3, "FULL OUTER"),
+    CROSS(4, "CROSS");
+
+    private final byte id;
+    private final String name;
+
+    JoinType(int id, String name) {
+        this.id = (byte) id;
+        this.name = name;
+    }
+
+    @Override
+    public String toString() {
+        return name;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeByte(id);
+    }
+
+    public static JoinType readFrom(StreamInput in) throws IOException {
+        byte id = in.readByte();
+        return switch (id) {
+            case 0 -> INNER;
+            case 1 -> LEFT;
+            case 2 -> RIGHT;
+            case 4 -> FULL;
+            case 5 -> CROSS;
+            default -> throw new IllegalArgumentException("unsupported join [" + id + "]");
+        };
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java
index 1ef8ca49b6e37..862098621e9ee 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java
@@ -10,7 +10,10 @@
 import org.elasticsearch.xpack.esql.core.plan.logical.LeafPlan;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
@@ -25,6 +28,18 @@ public LocalRelation(Source source, List<Attribute> output, LocalSupplier suppli
         this.supplier = supplier;
     }
 
+    public LocalRelation(PlanStreamInput in) throws IOException {
+        super(Source.readFrom(in));
+        this.output = in.readNamedWriteableCollectionAsList(Attribute.class);
+        this.supplier = LocalSupplier.readFrom(in);
+    }
+
+    public void writeTo(PlanStreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeNamedWriteableCollection(output);
+        supplier.writeTo(out);
+    }
+
     @Override
     protected NodeInfo<LocalRelation> info() {
         return NodeInfo.create(this, LocalRelation::new, output, supplier);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/HashJoinExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/HashJoinExec.java
new file mode 100644
index 0000000000000..7c124701fe332
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/HashJoinExec.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.plan.physical;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.io.stream.PlanNamedTypes;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+public class HashJoinExec extends UnaryExec implements EstimatesRowSize {
+    private final LocalSourceExec joinData;
+    private final List<NamedExpression> matchFields;
+    /**
+     * Conditions that must match for rows to be joined. The {@link Equals#left()}
+     * is always from the child and the {@link Equals#right()} is always from the
+     * {@link #joinData()}.
+     */
+    private final List<Equals> conditions;
+    private final List<Attribute> output;
+    private AttributeSet lazyAddedFields;
+
+    public HashJoinExec(
+        Source source,
+        PhysicalPlan child,
+        LocalSourceExec hashData,
+        List<NamedExpression> matchFields,
+        List<Equals> conditions,
+        List<Attribute> output
+    ) {
+        super(source, child);
+        this.joinData = hashData;
+        this.matchFields = matchFields;
+        this.conditions = conditions;
+        this.output = output;
+    }
+
+    public HashJoinExec(PlanStreamInput in) throws IOException {
+        super(Source.readFrom(in), in.readPhysicalPlanNode());
+        this.joinData = new LocalSourceExec(in);
+        this.matchFields = in.readNamedWriteableCollectionAsList(NamedExpression.class);
+        this.conditions = in.readCollectionAsList(i -> (Equals) PlanNamedTypes.readBinComparison(in, "equals"));
+        this.output = in.readNamedWriteableCollectionAsList(Attribute.class);
+    }
+
+    public void writeTo(PlanStreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writePhysicalPlanNode(child());
+        joinData.writeTo(out);
+        out.writeNamedWriteableCollection(matchFields);
+        out.writeCollection(conditions, (o, v) -> PlanNamedTypes.writeBinComparison(out, v));
+        out.writeNamedWriteableCollection(output);
+    }
+
+    public LocalSourceExec joinData() {
+        return joinData;
+    }
+
+    public List<NamedExpression> matchFields() {
+        return matchFields;
+    }
+
+    /**
+     * Conditions that must match for rows to be joined. The {@link Equals#left()}
+     * is always from the child and the {@link Equals#right()} is always from the
+     * {@link #joinData()}.
+     */
+    public List<Equals> conditions() {
+        return conditions;
+    }
+
+    public Set<Attribute> addedFields() {
+        if (lazyAddedFields == null) {
+            lazyAddedFields = outputSet();
+            lazyAddedFields.removeAll(child().output());
+        }
+        return lazyAddedFields;
+    }
+
+    @Override
+    public PhysicalPlan estimateRowSize(State state) {
+        state.add(false, output);
+        return this;
+    }
+
+    @Override
+    public List<Attribute> output() {
+        return output;
+    }
+
+    @Override
+    public HashJoinExec replaceChild(PhysicalPlan newChild) {
+        return new HashJoinExec(source(), newChild, joinData, matchFields, conditions, output);
+    }
+
+    @Override
+    protected NodeInfo<? extends PhysicalPlan> info() {
+        return NodeInfo.create(this, HashJoinExec::new, child(), joinData, matchFields, conditions, output);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        if (super.equals(o) == false) {
+            return false;
+        }
+        HashJoinExec hash = (HashJoinExec) o;
+        return joinData.equals(hash.joinData)
+            && matchFields.equals(hash.matchFields)
+            && conditions.equals(hash.conditions)
+            && output.equals(hash.output);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), joinData, matchFields, conditions, output);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LocalSourceExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LocalSourceExec.java
index 71c5955e31a1a..915e31bef7596 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LocalSourceExec.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LocalSourceExec.java
@@ -10,8 +10,11 @@
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
+import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
@@ -26,6 +29,18 @@ public LocalSourceExec(Source source, List<Attribute> output, LocalSupplier supp
         this.supplier = supplier;
     }
 
+    public LocalSourceExec(PlanStreamInput in) throws IOException {
+        super(Source.readFrom(in));
+        this.output = in.readNamedWriteableCollectionAsList(Attribute.class);
+        this.supplier = LocalSupplier.readFrom(in);
+    }
+
+    public void writeTo(PlanStreamOutput out) throws IOException {
+        source().writeTo(out);
+        out.writeNamedWriteableCollection(output);
+        supplier.writeTo(out);
+    }
+
     @Override
     public List<Attribute> output() {
         return output;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
index 3109575aa9c42..dc7e09dc8f174 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.lucene.LuceneOperator;
 import org.elasticsearch.compute.operator.ColumnExtractOperator;
+import org.elasticsearch.compute.operator.ColumnLoadOperator;
 import org.elasticsearch.compute.operator.Driver;
 import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator.EvalOperatorFactory;
@@ -28,6 +29,7 @@
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.compute.operator.Operator.OperatorFactory;
 import org.elasticsearch.compute.operator.OutputOperator.OutputOperatorFactory;
+import org.elasticsearch.compute.operator.RowInTableLookupOperator;
 import org.elasticsearch.compute.operator.RowOperator.RowOperatorFactory;
 import org.elasticsearch.compute.operator.ShowOperator;
 import org.elasticsearch.compute.operator.SinkOperator;
@@ -61,6 +63,7 @@
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupService;
 import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
 import org.elasticsearch.xpack.esql.evaluator.command.GrokEvaluatorExtracter;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.DissectExec;
 import org.elasticsearch.xpack.esql.plan.physical.EnrichExec;
@@ -73,6 +76,7 @@
 import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec;
 import org.elasticsearch.xpack.esql.plan.physical.FilterExec;
 import org.elasticsearch.xpack.esql.plan.physical.GrokExec;
+import org.elasticsearch.xpack.esql.plan.physical.HashJoinExec;
 import org.elasticsearch.xpack.esql.plan.physical.LimitExec;
 import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.MvExpandExec;
@@ -93,6 +97,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Function;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import static java.util.Arrays.asList;
@@ -218,6 +223,8 @@ else if (node instanceof EsQueryExec esQuery) {
         // lookups and joins
         else if (node instanceof EnrichExec enrich) {
             return planEnrich(enrich, context);
+        } else if (node instanceof HashJoinExec lookup) {
+            return planHashJoin(lookup, context);
         }
         // output
         else if (node instanceof OutputExec outputExec) {
@@ -480,6 +487,67 @@ private PhysicalOperation planEnrich(EnrichExec enrich, LocalExecutionPlannerCon
         );
     }
 
+    private PhysicalOperation planHashJoin(HashJoinExec join, LocalExecutionPlannerContext context) {
+        PhysicalOperation source = plan(join.child(), context);
+        int positionsChannel = source.layout.numberOfChannels();
+
+        Layout.Builder layoutBuilder = source.layout.builder();
+        for (Attribute f : join.output()) {
+            if (join.child().outputSet().contains(f)) {
+                continue;
+            }
+            layoutBuilder.append(f);
+        }
+        Layout layout = layoutBuilder.build();
+        Block[] localData = join.joinData().supplier().get();
+
+        RowInTableLookupOperator.Key[] keys = new RowInTableLookupOperator.Key[join.conditions().size()];
+        int[] blockMapping = new int[join.conditions().size()];
+        for (int k = 0; k < join.conditions().size(); k++) {
+            Equals cond = join.conditions().get(k);
+            Block localField = null;
+            for (int l = 0; l < join.joinData().output().size(); l++) {
+                if (join.joinData().output().get(l).name().equals((((NamedExpression) cond.right()).name()))) {
+                    localField = localData[l];
+                }
+            }
+            if (localField == null) {
+                throw new IllegalArgumentException("can't find local data for [" + cond.right() + "]");
+            }
+
+            NamedExpression left = (NamedExpression) cond.left();
+            keys[k] = new RowInTableLookupOperator.Key(left.name(), localField);
+            Layout.ChannelAndType input = source.layout.get(left.id());
+            blockMapping[k] = input.channel();
+        }
+
+        // Load the "positions" of each match
+        source = source.with(new RowInTableLookupOperator.Factory(keys, blockMapping), layout);
+
+        // Load the "values" from each match
+        for (Attribute f : join.addedFields()) {
+            Block localField = null;
+            for (int l = 0; l < join.joinData().output().size(); l++) {
+                if (join.joinData().output().get(l).name().equals(f.name())) {
+                    localField = localData[l];
+                }
+            }
+            if (localField == null) {
+                throw new IllegalArgumentException("can't find local data for [" + f + "]");
+            }
+            source = source.with(
+                new ColumnLoadOperator.Factory(new ColumnLoadOperator.Values(f.name(), localField), positionsChannel),
+                layout
+            );
+        }
+
+        // Drop the "positions" of the match
+        List<Integer> projection = new ArrayList<>();
+        IntStream.range(0, positionsChannel).boxed().forEach(projection::add);
+        IntStream.range(positionsChannel + 1, positionsChannel + 1 + join.addedFields().size()).boxed().forEach(projection::add);
+        return source.with(new ProjectOperatorFactory(projection), layout);
+    }
+
     private ExpressionEvaluator.Factory toEvaluator(Expression exp, Layout layout) {
         return EvalMapper.toEvaluator(exp, layout);
     }
@@ -522,6 +590,9 @@ private PhysicalOperation planProject(ProjectExec project, LocalExecutionPlanner
                 inputId = ne.id();
             }
             Layout.ChannelAndType input = source.layout.get(inputId);
+            if (input == null) {
+                throw new IllegalStateException("can't find input for [" + ne + "]");
+            }
             Layout.ChannelSet channelSet = inputChannelToOutputIds.get(input.channel());
             if (channelSet == null) {
                 channelSet = new Layout.ChannelSet(new HashSet<>(), input.type());
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java
index 12052b92432cf..9518954f78c64 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java
@@ -9,12 +9,15 @@
 
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.function.FunctionRegistry;
+import org.elasticsearch.xpack.esql.core.plan.logical.BinaryPlan;
 import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.core.plan.logical.Limit;
 import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy;
 import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
@@ -25,6 +28,8 @@
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.Row;
 import org.elasticsearch.xpack.esql.plan.logical.TopN;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinType;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.logical.meta.MetaFunctions;
 import org.elasticsearch.xpack.esql.plan.logical.show.ShowInfo;
@@ -37,6 +42,7 @@
 import org.elasticsearch.xpack.esql.plan.physical.FilterExec;
 import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
 import org.elasticsearch.xpack.esql.plan.physical.GrokExec;
+import org.elasticsearch.xpack.esql.plan.physical.HashJoinExec;
 import org.elasticsearch.xpack.esql.plan.physical.LimitExec;
 import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.MvExpandExec;
@@ -47,6 +53,9 @@
 import org.elasticsearch.xpack.esql.plan.physical.ShowExec;
 import org.elasticsearch.xpack.esql.plan.physical.TopNExec;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import static org.elasticsearch.xpack.esql.plan.physical.AggregateExec.Mode;
 import static org.elasticsearch.xpack.esql.plan.physical.AggregateExec.Mode.FINAL;
 import static org.elasticsearch.xpack.esql.plan.physical.AggregateExec.Mode.PARTIAL;
@@ -113,6 +122,24 @@ public PhysicalPlan map(LogicalPlan p) {
             return map(ua, child);
         }
 
+        if (p instanceof BinaryPlan bp) {
+            var left = map(bp.left());
+            var right = map(bp.right());
+
+            if (left instanceof FragmentExec) {
+                if (right instanceof FragmentExec) {
+                    throw new EsqlIllegalArgumentException("can't plan binary [" + p.nodeName() + "]");
+                }
+                // in case of a fragment, push to it any current streaming operator
+                return new FragmentExec(p);
+            }
+            if (right instanceof FragmentExec) {
+                // in case of a fragment, push to it any current streaming operator
+                return new FragmentExec(p);
+            }
+            return map(bp, left, right);
+        }
+
         throw new EsqlIllegalArgumentException("unsupported logical plan node [" + p.nodeName() + "]");
     }
 
@@ -181,7 +208,7 @@ private PhysicalPlan map(UnaryPlan p, PhysicalPlan child) {
             return map(aggregate, child);
         }
 
-        throw new EsqlIllegalArgumentException("unsupported unary logical plan node [" + p.nodeName() + "]");
+        throw new EsqlIllegalArgumentException("unsupported logical plan node [" + p.nodeName() + "]");
     }
 
     private PhysicalPlan map(Aggregate aggregate, PhysicalPlan child) {
@@ -239,4 +266,32 @@ private PhysicalPlan addExchangeForFragment(LogicalPlan logical, PhysicalPlan ch
         }
         return child;
     }
+
+    private PhysicalPlan map(BinaryPlan p, PhysicalPlan lhs, PhysicalPlan rhs) {
+        if (p instanceof Join join) {
+            PhysicalPlan hash = tryHashJoin(join, lhs, rhs);
+            if (hash != null) {
+                return hash;
+            }
+        }
+        throw new EsqlIllegalArgumentException("unsupported logical plan node [" + p.nodeName() + "]");
+    }
+
+    private PhysicalPlan tryHashJoin(Join join, PhysicalPlan lhs, PhysicalPlan rhs) {
+        if (join.config().type() != JoinType.LEFT) {
+            return null;
+        }
+        List<Equals> conditions = new ArrayList<>(join.config().conditions().size());
+        for (Expression cond : join.config().conditions()) {
+            if (cond instanceof Equals eq) {
+                conditions.add(eq);
+            } else {
+                return null;
+            }
+        }
+        if (rhs instanceof LocalSourceExec local) {
+            return new HashJoinExec(join.source(), lhs, local, join.config().matchFields(), conditions, join.output());
+        }
+        return null;
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
index 6328853eea3c6..b78958152b170 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.core.TimeValue;
@@ -274,6 +275,34 @@ public void testTablesLong() throws IOException {
         assertTablesOnlyValidOnSnapshot(request);
     }
 
+    public void testTablesDouble() throws IOException {
+        String json = """
+            {
+                "query": "ROW x = 1",
+                "tables": {"a": {"c:double": [1.1, 2, "3.1415", null, [5.1, "-6"]]}}
+            }
+            """;
+
+        EsqlQueryRequest request = parseEsqlQueryRequest(json, randomBoolean());
+        Column c = request.tables().get("a").get("c");
+        assertThat(c.type(), equalTo(DataType.DOUBLE));
+        try (
+            DoubleBlock.Builder builder = new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE)
+                .newDoubleBlockBuilder(10)
+        ) {
+            builder.appendDouble(1.1);
+            builder.appendDouble(2);
+            builder.appendDouble(3.1415);
+            builder.appendNull();
+            builder.beginPositionEntry();
+            builder.appendDouble(5.1);
+            builder.appendDouble(-6);
+            builder.endPositionEntry();
+            assertThat(c.values(), equalTo(builder.build()));
+        }
+        assertTablesOnlyValidOnSnapshot(request);
+    }
+
     public void testManyTables() throws IOException {
         String json = """
             {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
index 8adfc5813b855..975d8e1c7d7b8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
+import org.elasticsearch.xpack.esql.plan.logical.Lookup;
 import org.elasticsearch.xpack.esql.plan.logical.Row;
 import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
@@ -65,6 +66,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.elasticsearch.test.ListMatcher.matchesList;
+import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration;
@@ -78,8 +81,11 @@
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.matchesRegex;
+import static org.hamcrest.Matchers.startsWith;
 
 //@TestLogging(value = "org.elasticsearch.xpack.esql.analysis:TRACE", reason = "debug")
 public class AnalyzerTests extends ESTestCase {
@@ -1919,16 +1925,97 @@ public void testMvAppendValidation() {
     }
 
     public void testLookup() {
-        var e = expectThrows(ParsingException.class, () -> analyze("""
+        String query = """
               FROM test
             | RENAME languages AS int
             | LOOKUP int_number_names ON int
+            """;
+        if (Build.current().isProductionRelease()) {
+            var e = expectThrows(VerificationException.class, () -> analyze(query));
+            assertThat(e.getMessage(), containsString("line 3:4: LOOKUP is in preview and only available in SNAPSHOT build"));
+            return;
+        }
+        LogicalPlan plan = analyze(query);
+        var limit = as(plan, Limit.class);
+        assertThat(limit.limit().fold(), equalTo(1000));
+
+        var lookup = as(limit.child(), Lookup.class);
+        assertThat(lookup.tableName().fold(), equalTo("int_number_names"));
+        assertMap(lookup.matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}")));
+        assertThat(
+            lookup.localRelation().output().stream().map(Object::toString).toList(),
+            matchesList().item(startsWith("int{f}")).item(startsWith("name{f}"))
+        );
+
+        var project = as(lookup.child(), EsqlProject.class);
+        assertThat(project.projections().stream().map(Object::toString).toList(), hasItem(matchesRegex("languages\\{f}#\\d+ AS int#\\d+")));
+
+        var esRelation = as(project.child(), EsRelation.class);
+        assertThat(esRelation.index().name(), equalTo("test"));
+
+        // Lookup's output looks sensible too
+        assertMap(
+            lookup.output().stream().map(Object::toString).toList(),
+            matchesList().item(startsWith("_meta_field{f}"))
+                // TODO prune unused columns down through the join
+                .item(startsWith("emp_no{f}"))
+                .item(startsWith("first_name{f}"))
+                .item(startsWith("gender{f}"))
+                .item(startsWith("job{f}"))
+                .item(startsWith("job.raw{f}"))
+                /*
+                 * Int is a reference here because we renamed it in project.
+                 * If we hadn't it'd be a field and that'd be fine.
+                 */
+                .item(containsString("int{r}"))
+                .item(startsWith("last_name{f}"))
+                .item(startsWith("long_noidx{f}"))
+                .item(startsWith("salary{f}"))
+                /*
+                 * It's important that name is returned as a *reference* here
+                 * instead of a field. If it were a field we'd use SearchStats
+                 * on it and discover that it doesn't exist in the index. It doesn't!
+                 * We don't expect it to. It exists only in the lookup table.
+                 */
+                .item(containsString("name{r}"))
+        );
+    }
+
+    public void testLookupMissingField() {
+        var e = expectThrows(VerificationException.class, () -> analyze("""
+              FROM test
+            | LOOKUP int_number_names ON garbage
+            """));
+        if (Build.current().isProductionRelease()) {
+            assertThat(e.getMessage(), containsString("line 3:4: LOOKUP is in preview and only available in SNAPSHOT build"));
+            return;
+        }
+        assertThat(e.getMessage(), containsString("Unknown column in lookup target [garbage]"));
+    }
+
+    public void testLookupMissingTable() {
+        var e = expectThrows(VerificationException.class, () -> analyze("""
+              FROM test
+            | LOOKUP garbage ON a
+            """));
+        if (Build.current().isProductionRelease()) {
+            assertThat(e.getMessage(), containsString("line 3:4: LOOKUP is in preview and only available in SNAPSHOT build"));
+            return;
+        }
+        assertThat(e.getMessage(), containsString("Unknown table [garbage]"));
+    }
+
+    public void testLookupMatchTypeWrong() {
+        var e = expectThrows(VerificationException.class, () -> analyze("""
+              FROM test
+            | RENAME last_name AS int
+            | LOOKUP int_number_names ON int
             """));
         if (Build.current().isProductionRelease()) {
             assertThat(e.getMessage(), containsString("line 3:4: LOOKUP is in preview and only available in SNAPSHOT build"));
             return;
         }
-        assertThat(e.getMessage(), containsString("LOOKUP not yet supported"));
+        assertThat(e.getMessage(), containsString("column type mismatch, table column was [integer] and original column was [keyword]"));
     }
 
     public void testImplicitCasting() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
index 2abc70b4ecc88..b45fcd2724399 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java
@@ -73,10 +73,13 @@
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
 import org.elasticsearch.xpack.esql.plan.logical.Grok;
+import org.elasticsearch.xpack.esql.plan.logical.Lookup;
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.TopN;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
 import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.DissectExec;
 import org.elasticsearch.xpack.esql.plan.physical.EnrichExec;
@@ -90,7 +93,9 @@
 import org.elasticsearch.xpack.esql.plan.physical.FilterExec;
 import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
 import org.elasticsearch.xpack.esql.plan.physical.GrokExec;
+import org.elasticsearch.xpack.esql.plan.physical.HashJoinExec;
 import org.elasticsearch.xpack.esql.plan.physical.LimitExec;
+import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.MvExpandExec;
 import org.elasticsearch.xpack.esql.plan.physical.OrderExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
@@ -107,6 +112,8 @@
 import java.util.Set;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.test.ListMatcher.matchesList;
+import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.xpack.esql.SerializationTestUtils.serializeDeserialize;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -130,6 +137,8 @@ public class PlanNamedTypesTests extends ESTestCase {
         FragmentExec.class,
         GrokExec.class,
         LimitExec.class,
+        LocalSourceExec.class,
+        HashJoinExec.class,
         MvExpandExec.class,
         OrderExec.class,
         ProjectExec.class,
@@ -146,7 +155,7 @@ public void testPhysicalPlanEntries() {
             .filter(e -> e.categoryClass().isAssignableFrom(PhysicalPlan.class))
             .map(PlanNameRegistry.Entry::name)
             .toList();
-        assertThat(actual, equalTo(expected));
+        assertMap(actual, matchesList(expected));
     }
 
     // List of known serializable logical plan nodes - this should be kept up to date or retrieved
@@ -160,7 +169,10 @@ public void testPhysicalPlanEntries() {
         Eval.class,
         Filter.class,
         Grok.class,
+        Join.class,
         Limit.class,
+        LocalRelation.class,
+        Lookup.class,
         MvExpand.class,
         OrderBy.class,
         Project.class,
@@ -176,7 +188,7 @@ public void testLogicalPlanEntries() {
             .map(PlanNameRegistry.Entry::name)
             .sorted()
             .toList();
-        assertThat(actual, equalTo(expected));
+        assertMap(actual, matchesList(expected));
     }
 
     public void testFunctionEntries() {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
index f20f6eba671d3..b544bda70a500 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java
@@ -117,6 +117,8 @@
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.Row;
 import org.elasticsearch.xpack.esql.plan.logical.TopN;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinType;
 import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
@@ -135,6 +137,8 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singletonList;
+import static org.elasticsearch.test.ListMatcher.matchesList;
+import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.L;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
@@ -4968,6 +4972,158 @@ public void testIsNullDisjunction() throws Exception {
         assertEquals(and, new PropagateNullable().rule(and));
     }
 
+    //
+    // Lookup
+    //
+
+    /**
+     * Expects
+     * {@code
+     * Join[JoinConfig[type=LEFT OUTER, matchFields=[int{r}#4], conditions=[LOOKUP int_number_names ON int]]]
+     * |_EsqlProject[[_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, gender{f}#8, job{f}#13, job.raw{f}#14, languages{f}#9 AS int
+     * , last_name{f}#10, long_noidx{f}#15, salary{f}#11]]
+     * | \_Limit[1000[INTEGER]]
+     * |   \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..]
+     * \_LocalRelation[[int{f}#16, name{f}#17],[IntVectorBlock[vector=IntArrayVector[positions=10, values=[0, 1, 2, 3, 4, 5, 6, 7, 8,
+     * 9]]], BytesRefVectorBlock[vector=BytesRefArrayVector[positions=10]]]]
+     * }
+     */
+    public void testLookupSimple() {
+        var plan = optimizedPlan("""
+              FROM test
+            | RENAME languages AS int
+            | LOOKUP int_number_names ON int
+            """);
+        var join = as(plan, Join.class);
+
+        // Right is the lookup table
+        var right = as(join.right(), LocalRelation.class);
+        assertMap(
+            right.output().stream().map(Object::toString).sorted().toList(),
+            matchesList().item(containsString("int{f}")).item(containsString("name{f}"))
+        );
+
+        // Left is the rest of the query
+        var left = as(join.left(), EsqlProject.class);
+        assertThat(left.output().toString(), containsString("int{r}"));
+        var limit = as(left.child(), Limit.class);
+        assertThat(limit.limit().fold(), equalTo(1000));
+
+        assertThat(join.config().type(), equalTo(JoinType.LEFT));
+        assertThat(join.config().matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}")));
+        assertThat(join.config().conditions().size(), equalTo(1));
+        Equals eq = as(join.config().conditions().get(0), Equals.class);
+        assertThat(eq.left().toString(), startsWith("int{r}"));
+        assertThat(eq.right().toString(), startsWith("int{r}"));
+        assertTrue(join.children().get(0).outputSet() + " contains " + eq.left(), join.children().get(0).outputSet().contains(eq.left()));
+        assertTrue(join.children().get(1).outputSet() + " contains " + eq.right(), join.children().get(1).outputSet().contains(eq.right()));
+
+        // Join's output looks sensible too
+        assertMap(
+            join.output().stream().map(Object::toString).toList(),
+            matchesList().item(startsWith("_meta_field{f}"))
+                // TODO prune unused columns down through the join
+                .item(startsWith("emp_no{f}"))
+                .item(startsWith("first_name{f}"))
+                .item(startsWith("gender{f}"))
+                .item(startsWith("job{f}"))
+                .item(startsWith("job.raw{f}"))
+                /*
+                 * Int is a reference here because we renamed it in project.
+                 * If we hadn't it'd be a field and that'd be fine.
+                 */
+                .item(containsString("int{r}"))
+                .item(startsWith("last_name{f}"))
+                .item(startsWith("long_noidx{f}"))
+                .item(startsWith("salary{f}"))
+                /*
+                 * It's important that name is returned as a *reference* here
+                 * instead of a field. If it were a field we'd use SearchStats
+                 * on it and discover that it doesn't exist in the index. It doesn't!
+                 * We don't expect it to. It exists only in the lookup table.
+                 */
+                .item(containsString("name{r}"))
+        );
+    }
+
+    /**
+     * Expects
+     * {@code
+     * Limit[1000[INTEGER]]
+     * \_Aggregate[[name{r}#20],[MIN(emp_no{f}#9) AS MIN(emp_no), name{r}#20]]
+     *   \_Join[JoinConfig[type=LEFT OUTER, matchFields=[int{r}#4], conditions=[LOOKUP int_number_names ON int]]]
+     *     |_EsqlProject[[_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, gender{f}#11, job{f}#16, job.raw{f}#17, languages{f}#12 AS
+     * int, last_name{f}#13, long_noidx{f}#18, salary{f}#14]]
+     *     | \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..]
+     *     \_LocalRelation[[int{f}#19, name{f}#20],[IntVectorBlock[vector=IntArrayVector[positions=10, values=[0, 1, 2, 3, 4, 5, 6, 7, 8,
+     * 9]]], BytesRefVectorBlock[vector=BytesRefArrayVector[positions=10]]]]
+     * }
+     */
+    public void testLookupStats() {
+        var plan = optimizedPlan("""
+              FROM test
+            | RENAME languages AS int
+            | LOOKUP int_number_names ON int
+            | STATS MIN(emp_no) BY name
+            """);
+        var limit = as(plan, Limit.class);
+        assertThat(limit.limit().fold(), equalTo(1000));
+
+        var agg = as(limit.child(), Aggregate.class);
+        assertMap(
+            agg.aggregates().stream().map(Object::toString).sorted().toList(),
+            matchesList().item(startsWith("MIN(emp_no)")).item(startsWith("name{r}"))
+        );
+        assertMap(agg.groupings().stream().map(Object::toString).toList(), matchesList().item(startsWith("name{r}")));
+
+        var join = as(agg.child(), Join.class);
+        // Right is the lookup table
+        var right = as(join.right(), LocalRelation.class);
+        assertMap(
+            right.output().stream().map(Object::toString).toList(),
+            matchesList().item(containsString("int{f}")).item(containsString("name{f}"))
+        );
+
+        // Left is the rest of the query
+        var left = as(join.left(), EsqlProject.class);
+        assertThat(left.output().toString(), containsString("int{r}"));
+        as(left.child(), EsRelation.class);
+
+        assertThat(join.config().type(), equalTo(JoinType.LEFT));
+        assertThat(join.config().matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}")));
+        assertThat(join.config().conditions().size(), equalTo(1));
+        Equals eq = as(join.config().conditions().get(0), Equals.class);
+        assertThat(eq.left().toString(), startsWith("int{r}"));
+        assertThat(eq.right().toString(), startsWith("int{r}"));
+
+        // Join's output looks sensible too
+        assertMap(
+            join.output().stream().map(Object::toString).toList(),
+            matchesList().item(startsWith("_meta_field{f}"))
+                // TODO prune unused columns down through the join
+                .item(startsWith("emp_no{f}"))
+                .item(startsWith("first_name{f}"))
+                .item(startsWith("gender{f}"))
+                .item(startsWith("job{f}"))
+                .item(startsWith("job.raw{f}"))
+                /*
+                 * Int is a reference here because we renamed it in project.
+                 * If we hadn't it'd be a field and that'd be fine.
+                 */
+                .item(containsString("int{r}"))
+                .item(startsWith("last_name{f}"))
+                .item(startsWith("long_noidx{f}"))
+                .item(startsWith("salary{f}"))
+                /*
+                 * It's important that name is returned as a *reference* here
+                 * instead of a field. If it were a field we'd use SearchStats
+                 * on it and discover that it doesn't exist in the index. It doesn't!
+                 * We don't expect it to. It exists only in the lookup table.
+                 */
+                .item(containsString("name{r}"))
+        );
+    }
+
     private Literal nullOf(DataType dataType) {
         return new Literal(Source.EMPTY, null, dataType);
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
index 280351b01fff7..bc70ce64944d1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
@@ -47,6 +47,7 @@
 import org.elasticsearch.xpack.esql.core.index.IndexResolution;
 import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.core.plan.logical.Limit;
+import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
@@ -73,7 +74,11 @@
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
+import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.TopN;
+import org.elasticsearch.xpack.esql.plan.logical.join.Join;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinType;
+import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.esql.plan.physical.DissectExec;
@@ -88,6 +93,7 @@
 import org.elasticsearch.xpack.esql.plan.physical.FilterExec;
 import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
 import org.elasticsearch.xpack.esql.plan.physical.GrokExec;
+import org.elasticsearch.xpack.esql.plan.physical.HashJoinExec;
 import org.elasticsearch.xpack.esql.plan.physical.LimitExec;
 import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
@@ -108,6 +114,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -115,6 +122,8 @@
 import static org.elasticsearch.core.Tuple.tuple;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
+import static org.elasticsearch.test.ListMatcher.matchesList;
+import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration;
@@ -135,10 +144,14 @@
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.matchesRegex;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.startsWith;
 
 // @TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug")
 public class PhysicalPlanOptimizerTests extends ESTestCase {
@@ -4079,6 +4092,145 @@ public void testMaxQueryDepthPlusExpressionDepth() {
         assertThat(e.getMessage(), containsString("ESQL statement exceeded the maximum query depth allowed (" + MAX_QUERY_DEPTH + ")"));
     }
 
+    public void testLookupSimple() {
+        PhysicalPlan plan = physicalPlan("""
+              FROM test |
+            RENAME languages AS int |
+            LOOKUP int_number_names ON int""");
+        var join = as(plan, HashJoinExec.class);
+        assertMap(join.matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}")));
+        assertMap(
+            join.output().stream().map(Object::toString).toList(),
+            matchesList().item(startsWith("_meta_field{f}"))
+                .item(startsWith("emp_no{f}"))
+                .item(startsWith("first_name{f}"))
+                .item(startsWith("gender{f}"))
+                .item(startsWith("job{f}"))
+                .item(startsWith("job.raw{f}"))
+                .item(startsWith("int{r}"))
+                .item(startsWith("last_name{f}"))
+                .item(startsWith("long_noidx{f}"))
+                .item(startsWith("salary{f}"))
+                .item(startsWith("name{r}"))
+        );
+    }
+
+    /**
+     * Expected
+     * {@code
+     * ProjectExec[[emp_no{f}#17, int{r}#5 AS languages, name{f}#28 AS lang_name]]
+     * \_HashJoinExec[
+     *      LocalSourceExec[[int{f}#27, name{f}#28],[...]],
+     *      [int{r}#5],
+     *      [name{r}#28, _meta_field{f}#23, emp_no{f}#17, ...]]
+     *   \_ProjectExec[[_meta_field{f}#23, emp_no{f}#17, ...]]
+     *     \_TopNExec[[Order[emp_no{f}#17,ASC,LAST]],4[INTEGER],370]
+     *       \_ExchangeExec[[],false]
+     *         \_ProjectExec[[emp_no{f}#17, ..., languages{f}#20]]
+     *           \_FieldExtractExec[emp_no{f}#17, _meta_field{f}#23, first_name{f}#18, ..]<[]>
+     *             \_EsQueryExec[...]
+     * }
+     */
+    public void testLookupThenProject() {
+        PhysicalPlan plan = optimizedPlan(physicalPlan("""
+            FROM employees
+            | SORT emp_no
+            | LIMIT 4
+            | RENAME languages AS int
+            | LOOKUP int_number_names ON int
+            | RENAME int AS languages, name AS lang_name
+            | KEEP emp_no, languages, lang_name"""));
+
+        var outerProject = as(plan, ProjectExec.class);
+        assertThat(outerProject.projections().toString(), containsString("AS lang_name"));
+        var join = as(outerProject.child(), HashJoinExec.class);
+        assertMap(join.matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}")));
+        assertMap(
+            join.output().stream().map(Object::toString).toList(),
+            matchesList().item(startsWith("_meta_field{f}"))
+                .item(startsWith("emp_no{f}"))
+                .item(startsWith("first_name{f}"))
+                .item(startsWith("gender{f}"))
+                .item(startsWith("job{f}"))
+                .item(startsWith("job.raw{f}"))
+                .item(startsWith("int{r}"))
+                .item(startsWith("last_name{f}"))
+                .item(startsWith("long_noidx{f}"))
+                .item(startsWith("salary{f}"))
+                .item(startsWith("name{r}"))
+        );
+
+        var middleProject = as(join.child(), ProjectExec.class);
+        assertThat(middleProject.projections().stream().map(Objects::toString).toList(), not(hasItem(startsWith("name{f}"))));
+        /*
+         * At the moment we don't push projections past the HashJoin so we still include first_name here
+         */
+        assertThat(middleProject.projections().stream().map(Objects::toString).toList(), hasItem(startsWith("first_name{f}")));
+
+        var outerTopn = as(middleProject.child(), TopNExec.class);
+        var exchange = as(outerTopn.child(), ExchangeExec.class);
+        var innerProject = as(exchange.child(), ProjectExec.class);
+        assertThat(innerProject.projections().stream().map(Objects::toString).toList(), not(hasItem(startsWith("name{f}"))));
+    }
+
+    /**
+     * Expects optimized data node plan of
+     * <pre>{@code
+     * TopN[[Order[name{r}#25,ASC,LAST], Order[emp_no{f}#14,ASC,LAST]],1000[INTEGER]]
+     * \_Join[JoinConfig[type=LEFT OUTER, unionFields=[int{r}#4]]]
+     *   |_EsqlProject[[..., long_noidx{f}#23, salary{f}#19]]
+     *   | \_EsRelation[test][_meta_field{f}#20, emp_no{f}#14, first_name{f}#15, ..]
+     *   \_LocalRelation[[int{f}#24, name{f}#25],[...]]
+     * }</pre>
+     */
+    public void testLookupThenTopN() {
+        var plan = physicalPlan("""
+                  FROM employees
+                | RENAME languages AS int
+                | LOOKUP int_number_names ON int
+                | RENAME name AS languages
+                | KEEP languages, emp_no
+                | SORT languages ASC, emp_no ASC
+            """);
+
+        ProjectExec outerProject = as(plan, ProjectExec.class);
+        TopNExec outerTopN = as(outerProject.child(), TopNExec.class);
+        ExchangeExec exchange = as(outerTopN.child(), ExchangeExec.class);
+        FragmentExec frag = as(exchange.child(), FragmentExec.class);
+
+        LogicalPlan opt = logicalOptimizer.optimize(frag.fragment());
+        TopN innerTopN = as(opt, TopN.class);
+        assertMap(
+            innerTopN.order().stream().map(o -> o.child().toString()).toList(),
+            matchesList().item(startsWith("name{r}")).item(startsWith("emp_no{f}"))
+        );
+        Join join = as(innerTopN.child(), Join.class);
+        assertThat(join.config().type(), equalTo(JoinType.LEFT));
+        assertMap(join.config().matchFields().stream().map(Objects::toString).toList(), matchesList().item(startsWith("int{r}")));
+
+        Project innerProject = as(join.left(), Project.class);
+        assertThat(innerProject.projections(), hasSize(10));
+        assertMap(
+            innerProject.projections().stream().map(Object::toString).toList(),
+            matchesList().item(startsWith("_meta_field{f}"))
+                .item(startsWith("emp_no{f}"))
+                .item(startsWith("first_name{f}"))
+                .item(startsWith("gender{f}"))
+                .item(startsWith("job{f}"))
+                .item(startsWith("job.raw{f}"))
+                .item(matchesRegex("languages\\{f}#\\d+ AS int#\\d+"))
+                .item(startsWith("last_name{f}"))
+                .item(startsWith("long_noidx{f}"))
+                .item(startsWith("salary{f}"))
+        );
+
+        LocalRelation lookup = as(join.right(), LocalRelation.class);
+        assertMap(
+            lookup.output().stream().map(Object::toString).toList(),
+            matchesList().item(startsWith("int{f}")).item(startsWith("name{f}"))
+        );
+    }
+
     @SuppressWarnings("SameParameterValue")
     private static void assertFilterCondition(
         Filter filter,
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
index b30b9c1598397..884b24fc0fc57 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
@@ -48,6 +48,7 @@
 import org.elasticsearch.xpack.esql.plan.logical.Explain;
 import org.elasticsearch.xpack.esql.plan.logical.Grok;
 import org.elasticsearch.xpack.esql.plan.logical.InlineStats;
+import org.elasticsearch.xpack.esql.plan.logical.Lookup;
 import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
 import org.elasticsearch.xpack.esql.plan.logical.Project;
 import org.elasticsearch.xpack.esql.plan.logical.Row;
@@ -945,6 +946,16 @@ public void testInlineConvertWithNonexistentType() {
         expectError("ROW (1+2)::doesnotexist", "line 1:13: Unknown data type named [doesnotexist]");
     }
 
+    public void testLookup() {
+        var plan = statement("ROW a = 1 | LOOKUP t ON j");
+        var lookup = as(plan, Lookup.class);
+        var tableName = as(lookup.tableName(), Literal.class);
+        assertThat(tableName.fold(), equalTo("t"));
+        assertThat(lookup.matchFields(), hasSize(1));
+        var matchField = as(lookup.matchFields().get(0), UnresolvedAttribute.class);
+        assertThat(matchField.name(), equalTo("j"));
+    }
+
     public void testInlineConvertUnsupportedType() {
         expectError("ROW 3::BYTE", "line 1:6: Unsupported conversion to type [BYTE]");
     }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java
index 5c96c410f27ec..b08a2798bc509 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java
@@ -165,6 +165,7 @@ private class TestFieldExtractOperator implements Operator {
         private final MappedFieldType.FieldExtractPreference extractPreference;
 
         TestFieldExtractOperator(String columnName, DataType dataType, MappedFieldType.FieldExtractPreference extractPreference) {
+            assert columnNames.contains(columnName);
             this.columnName = columnName;
             this.dataType = dataType;
             this.extractPreference = extractPreference;
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
index a8b371bcdd608..e50ba59a31b2d 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.Order;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedNamedExpression;
@@ -30,6 +31,8 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat;
 import org.elasticsearch.xpack.esql.plan.logical.Dissect;
 import org.elasticsearch.xpack.esql.plan.logical.Grok;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig;
+import org.elasticsearch.xpack.esql.plan.logical.join.JoinType;
 import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.Stat;
 import org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType;
@@ -83,6 +86,12 @@ protected Object pluggableMakeArg(Class<? extends Node<?>> toBuildClass, Class<?
             return new Stat(randomRealisticUnicodeOfLength(10), randomFrom(StatsType.values()), null);
         } else if (argClass == Integer.class) {
             return randomInt();
+        } else if (argClass == JoinConfig.class) {
+            return new JoinConfig(
+                randomFrom(JoinType.values()),
+                randomList(0, 10, () -> (NamedExpression) makeArg(NamedExpression.class)),
+                randomList(0, 10, () -> (Expression) makeArg(Expression.class))
+            );
         }
 
         return null;
@@ -118,7 +127,7 @@ protected Predicate<String> pluggableClassNameFilter() {
     }
 
     /** Scans the {@code .class} files to identify all classes and checks if they are subclasses of {@link Node}. */
-    @ParametersFactory
+    @ParametersFactory(argumentFormatting = "%1s")
     @SuppressWarnings("rawtypes")
     public static List<Object[]> nodeSubclasses() throws IOException {
         return subclassesOf(Node.class, CLASSNAME_FILTER).stream()
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/RuleExecutor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/RuleExecutor.java
index 2ccdd66089c79..b0fad3da6b036 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/RuleExecutor.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/RuleExecutor.java
@@ -171,7 +171,7 @@ protected final ExecutionInfo executeWithInfo(TreeType plan) {
                     if (tf.hasChanged()) {
                         hasChanged = true;
                         if (log.isTraceEnabled()) {
-                            log.trace("Rule {} applied\n{}", rule, NodeUtils.diffString(tf.before, tf.after));
+                            log.trace("Rule {} applied with changes\n{}", rule, NodeUtils.diffString(tf.before, tf.after));
                         }
                     } else {
                         if (log.isTraceEnabled()) {
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml
index e8b372e6d7e8e..dcffca2f28fb0 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml
@@ -1,32 +1,210 @@
-index named lookup:
+---
+setup:
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            number_of_shards: 5
+          mappings:
+            properties:
+              data:
+                type: long
+              data_d:
+                type: double
+              count:
+                type: long
+              count_d:
+                type: double
+              time:
+                type: long
+              color:
+                type: keyword
   - do:
       bulk:
-        index: lookup
+        index: "test"
         refresh: true
         body:
-          - { index: { } }
-          - { f: 1 }
+          - { "index": { } }
+          - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275187, "color": "red" }
+          - { "index": { } }
+          - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275188, "color": "blue" }
+
+---
+basic:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [lookup_command]
+      reason: "uses LOOKUP"
 
   - do:
       esql.query:
         body:
-          query: 'FROM lookup | LIMIT 1'
-  - match: { columns.0.name: f }
-  - match: { columns.0.type: long }
-  - length: { values: 1 }
-  - match: { values.0.0: 1 }
+          query: 'FROM test | SORT time | KEEP color | LOOKUP colors ON color | LIMIT 2'
+          columnar: true
+          tables:
+            colors:
+              "color:keyword":  ["red", "green", "blue"]
+              "rgb:integer": [16711680,   65280,    255]
+
+  - match: {columns.0.name: "color"}
+  - match: {columns.0.type: "keyword"}
+  - match: {columns.1.name: "rgb"}
+  - match: {columns.1.type: "integer"}
+  - match: {values.0: ["red", "blue"]}
+  - match: {values.1: [16711680, 255]}
 
 ---
-lookup command unsupported:
+read multivalue keyword:
   - requires:
       test_runner_features: [capabilities]
       capabilities:
         - method: POST
           path: /_query
-          parameters: [method, path, parameters, capabilities]
-          capabilities: [lookup]
-      reason: "LOOKUP command required"
+          parameters: []
+          capabilities: [lookup_command]
+      reason: "uses LOOKUP"
 
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test | SORT time | KEEP color | LOOKUP color_associations ON color | LIMIT 2'
+          columnar: true
+          tables:
+            color_associations:
+              "color:keyword":  ["red", "green", "blue"]
+              "association:keyword":
+                - ["love", "passion", "blood", "happiness"]
+                - ["nature", "healing", "health", "youth"]
+                - ["serenity", "wisdom", "ocean", "sky"]
+
+  - match: {columns.0.name: "color"}
+  - match: {columns.0.type: "keyword"}
+  - match: {columns.1.name: "association"}
+  - match: {columns.1.type: "keyword"}
+  - match: {values.0: ["red", "blue"]}
+  - match: {values.1: [["love", "passion", "blood", "happiness"], ["serenity", "wisdom", "ocean", "sky"]]}
+
+---
+keyword matches text:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [lookup_command]
+      reason: "uses LOOKUP"
+
+  - do:
+      indices.create:
+        index:  test_text
+        body:
+          settings:
+            number_of_shards: 1
+          mappings:
+            properties:
+              color:
+                type: text
+  - do:
+      bulk:
+        index: test_text
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "color": "red" }
+
+  - do:
+      esql.query:
+        body:
+          query: 'FROM test_text | LOOKUP colors ON color | LIMIT 1'
+          columnar: true
+          tables:
+            colors:
+              "color:keyword":  ["red", "green", "blue"]
+              "rgb:integer": [16711680,   65280,    255]
+
+  - match: {columns.0.name: "color"}
+  - match: {columns.0.type: "text"}
+  - match: {columns.1.name: "rgb"}
+  - match: {columns.1.type: "integer"}
+  - match: {values.0: ["red"]}
+  - match: {values.1: [16711680]}
+
+---
+duplicate column names in table:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [lookup_command]
+      reason: "uses LOOKUP"
+
+  - do:
+      catch: /duplicate column name \[color\]/
+      esql.query:
+        body:
+          query: 'FROM test | LOOKUP colors ON color | SORT time | KEEP color, rgb | LIMIT 2'
+          columnar: true
+          tables:
+            colors:
+              "color:keyword":    ["red", "green", "blue"]
+              "color:integer": [16711680,   65280,    255]
+
+---
+duplicate keys:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [lookup_command]
+      reason: "uses LOOKUP"
+
+  - do:
+      # TODO improve this error message
+      catch: /found a duplicate row/
+      esql.query:
+        body:
+          query: 'FROM test | LOOKUP colors ON color | SORT time | KEEP color, rgb | LIMIT 2'
+          columnar: true
+          tables:
+            colors:
+              "color:keyword":  ["red",   "red", "blue"]
+              "rgb:integer": [16711680,   65280,    255]
+
+---
+multivalued keys:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [lookup_command]
+      reason: "uses LOOKUP"
+
+  - do:
+      # TODO improve this error message
+      catch: /only single valued keys are supported/
+      esql.query:
+        body:
+          query: 'FROM test | LOOKUP colors ON color | SORT time | KEEP color, rgb | LIMIT 2'
+          columnar: true
+          tables:
+            colors:
+              "color:keyword":  [["red", "blue"],   "white", "blue"]
+              "rgb:integer":           [16711680,     65280,    255]
+
+---
+index named lookup still works:
   - do:
       bulk:
         index: lookup
@@ -36,7 +214,32 @@ lookup command unsupported:
           - { f: 1 }
 
   - do:
-      catch: /LOOKUP not yet supported/
       esql.query:
         body:
-          query: 'FROM lookup | LOOKUP a ON foo'
+          query: 'FROM lookup | LIMIT 1'
+  - match: { columns.0.name: f }
+  - match: { columns.0.type: long }
+  - length: { values: 1 }
+  - match: { values.0.0: 1 }
+
+---
+on function:
+  - requires:
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [lookup_command]
+      reason: "uses LOOKUP"
+
+  - do:
+      catch: "/token recognition error at: '\\('/"
+      esql.query:
+        body:
+          query: 'FROM test | SORT time | KEEP color | LOOKUP colors ON CONCAT(color, "foo") | LIMIT 2'
+          columnar: true
+          tables:
+            colors:
+              "color:keyword":  ["red", "green", "blue"]
+              "rgb:integer": [16711680,   65280,    255]

From 1c3b3d8f110c50c5c3e063c018330369f75199d3 Mon Sep 17 00:00:00 2001
From: Panagiotis Bailis <pmpailis@gmail.com>
Date: Fri, 7 Jun 2024 11:09:06 +0300
Subject: [PATCH 207/208] Adding support for explain in rrf (#108682)

---
 docs/changelog/108682.yaml                    |   5 +
 docs/reference/search/rrf.asciidoc            | 156 ++++++-
 .../PercolatorHighlightSubFetchPhase.java     |   3 +-
 ...rcolatorMatchedSlotSubFetchPhaseTests.java |   6 +-
 .../search/rank/FieldBasedRerankerIT.java     |   6 +
 .../org/elasticsearch/TransportVersions.java  |   1 +
 .../action/search/FetchSearchPhase.java       |  45 +-
 .../action/search/SearchRequest.java          |   3 -
 .../elasticsearch/search/SearchModule.java    |   7 +
 .../elasticsearch/search/SearchService.java   |   6 +-
 .../metrics/TopHitsAggregator.java            |   2 +-
 .../search/fetch/FetchContext.java            |  14 +
 .../search/fetch/FetchPhase.java              |  33 +-
 .../search/fetch/FetchSubPhase.java           |  18 +-
 .../search/fetch/ShardFetchRequest.java       |   6 +
 .../search/fetch/ShardFetchSearchRequest.java |  18 +
 .../search/fetch/subphase/ExplainPhase.java   |   7 +
 .../search/fetch/subphase/InnerHitsPhase.java |   2 +-
 .../search/rank/RankBuilder.java              |   8 +
 .../elasticsearch/search/rank/RankDoc.java    |   6 +-
 .../search/rank/RankDocShardInfo.java         |  44 ++
 .../search/rank/feature/RankFeatureDoc.java   |   7 +
 .../search/retriever/KnnRetrieverBuilder.java |   3 +
 .../search/retriever/RetrieverBuilder.java    |   9 +
 .../action/search/RankFeaturePhaseTests.java  |   8 +
 .../action/search/SearchRequestTests.java     |  12 -
 .../search/SearchServiceTests.java            |   1 +
 .../fetch/subphase/FetchFieldsPhaseTests.java |   2 +-
 .../fetch/subphase/FetchSourcePhaseTests.java |   2 +-
 .../rank/RankFeatureShardPhaseTests.java      |   7 +
 .../search/fetch/HighlighterTestCase.java     |   3 +-
 .../search/rank/TestRankBuilder.java          |   6 +
 .../search/rank/TestRankDoc.java              |   5 +
 .../xpack/rank/rrf/RRFRankMultiShardIT.java   | 232 ++++++++++-
 .../xpack/rank/rrf/RRFRankBuilder.java        |  59 +++
 .../xpack/rank/rrf/RRFRankDoc.java            |   7 +
 .../xpack/rank/rrf/RRFRankPlugin.java         |   4 +-
 .../test/rrf/500_rrf_retriever_explain.yml    | 275 +++++++++++++
 .../test/rrf/550_rrf_sub_searches_explain.yml | 386 ++++++++++++++++++
 39 files changed, 1377 insertions(+), 47 deletions(-)
 create mode 100644 docs/changelog/108682.yaml
 create mode 100644 server/src/main/java/org/elasticsearch/search/rank/RankDocShardInfo.java
 create mode 100644 x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/500_rrf_retriever_explain.yml
 create mode 100644 x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml

diff --git a/docs/changelog/108682.yaml b/docs/changelog/108682.yaml
new file mode 100644
index 0000000000000..bd566acab8306
--- /dev/null
+++ b/docs/changelog/108682.yaml
@@ -0,0 +1,5 @@
+pr: 108682
+summary: Adding support for explain in rrf
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/reference/search/rrf.asciidoc b/docs/reference/search/rrf.asciidoc
index 04f698f06d879..ba0f6c018b0eb 100644
--- a/docs/reference/search/rrf.asciidoc
+++ b/docs/reference/search/rrf.asciidoc
@@ -115,7 +115,6 @@ The `rrf` retriever does not currently support:
 * <<search-suggesters, suggesters>>
 * <<highlighting, highlighting>>
 * <<collapse-search-results, collapse>>
-* <<request-body-search-explain, explain>>
 * <<profiling-queries, profiling>>
 
 Using unsupported features as part of a search with an `rrf` retriever results
@@ -519,6 +518,161 @@ We end with `_id: 3` as `_rank: 1`, `_id: 2` as `_rank: 2`, and
 `_id: 4` as `_rank: 3`. This ranking matches the result set from the
 original RRF search as expected.
 
+==== Explain in RRF
+
+In addition to individual query scoring details, we can make use of the `explain=true` parameter to get information on how
+the RRF scores for each document were computed. Working with the example above, and by adding
+`explain=true` to the search request, we'd now have a response that looks like the following:
+
+[source,js]
+----
+{
+    "hits":
+    [
+        {
+            "_index": "example-index",
+            "_id": "3",
+            "_score": 0.8333334,
+            "_rank": 1,
+            "_explanation":
+            {
+                "value": 0.8333334,                                                                                                                                               <1>
+                "description": "rrf score: [0.8333334] computed for initial ranks [2, 1] with rankConstant: [1] as sum of [1 / (rank + rankConstant)] for each query",            <2>
+                "details":                                                                                                                                                        <3>
+                [
+                    {
+                        "value": 2,                                                                                                                                               <4>
+                        "description": "rrf score: [0.33333334], for rank [2] in query at index [0] computed as [1 / (2 + 1]), for matching query with score: ",
+                        "details":                                                                                                                                                <5>
+                        [
+                            {
+                                "value": 0.15876243,
+                                "description": "weight(text:rrf in 0) [PerFieldSimilarity], result of:",
+                                "details":
+                                [
+                                    ...
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        "value": 1,                                                                                                                                              <6>
+                        "description": "rrf score: [0.5], for rank [1] in query at index [1] computed as [1 / (1 + 1]), for matching query with score: ",
+                        "details":
+                        [
+                            {
+                                "value": 1,
+                                "description": "within top k documents",
+                                "details":
+                                []
+                            }
+                        ]
+                    }
+                ]
+            }
+        }
+        ...
+    ]
+}
+----
+// NOTCONSOLE
+
+<1> the final RRF score for document with `_id=3`
+<2> a description on how this score was computed based on the ranks of this document in each individual query
+<3> details on how the RRF score was computed for each of the queries
+<4> the `value` heres specifies the `rank` of this document in the specific query
+<5> standard `explain` output of the underlying query, describing matching terms and weights
+<6> the `value` heres specifies the `rank` of this document for the second (`knn`) query
+
+In addition to the above, explain in RRF also supports <<named-queries,named queries>> using the `_name` parameter.
+Using named queries allows for easier and more intuitive understanding of the RRF score computation,
+especially when dealing with multiple queries. So, we would now have:
+
+[source,js]
+----
+GET example-index/_search
+{
+    "retriever": {
+        "rrf": {
+            "retrievers": [
+                {
+                    "standard": {
+                        "query": {
+                            "term": {
+                                "text": "rrf"
+                            }
+                        }
+                    }
+                },
+                {
+                    "knn": {
+                        "field": "vector",
+                        "query_vector": [3],
+                        "k": 5,
+                        "num_candidates": 5,
+                        "_name": "my_knn_query"                           <1>
+                    }
+                }
+            ],
+            "rank_window_size": 5,
+            "rank_constant": 1
+        }
+    },
+    "size": 3,
+    "aggs": {
+        "int_count": {
+            "terms": {
+                "field": "integer"
+            }
+        }
+    }
+}
+----
+// NOTCONSOLE
+<1> Here we specify a `_name` for the `knn` retriever
+
+The response would now include the named query in the explanation:
+[source,js]
+----
+{
+    "hits":
+    [
+        {
+            "_index": "example-index",
+            "_id": "3",
+            "_score": 0.8333334,
+            "_rank": 1,
+            "_explanation":
+            {
+                "value": 0.8333334,
+                "description": "rrf score: [0.8333334] computed for initial ranks [2, 1] with rankConstant: [1] as sum of [1 / (rank + rankConstant)] for each query",
+                "details":
+                [
+                    {
+                        "value": 2,
+                        "description": "rrf score: [0.33333334], for rank [2] in query at index [0] computed as [1 / (2 + 1]), for matching query with score: ",
+                        "details":
+                        [
+                            ...
+                        ]
+                    },
+                    {
+                        "value": 1,
+                        "description": "rrf score: [0.5], for rank [1] in query [my_knn_query] computed as [1 / (1 + 1]), for matching query with score: ",                      <1>
+                        "details":
+                        [
+                           ...
+                        ]
+                    }
+                ]
+            }
+        }
+        ...
+    ]
+}
+----
+// NOTCONSOLE
+<1> Instead of the anonymous `at index n` , we now have a reference to the named query `my_knn_query`.
 
 ==== Pagination in RRF
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
index 138007c104d2b..da8d7de27d317 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
@@ -101,7 +101,8 @@ public void process(HitContext hit) throws IOException {
                                 percolatorLeafReaderContext,
                                 slot,
                                 leafStoredFields.storedFields(),
-                                Source.fromBytes(document)
+                                Source.fromBytes(document),
+                                null
                             );
                             processor.process(subContext);
                             for (Map.Entry<String, HighlightField> entry : subContext.hit().getHighlightFields().entrySet()) {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java
index 82ec63b785e56..1ebf0b4a28ed6 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java
@@ -56,7 +56,7 @@ public void testHitsExecute() throws Exception {
                 LeafReaderContext context = reader.leaves().get(0);
                 // A match:
                 {
-                    HitContext hit = new HitContext(SearchHit.unpooled(0), context, 0, Map.of(), Source.empty(null));
+                    HitContext hit = new HitContext(SearchHit.unpooled(0), context, 0, Map.of(), Source.empty(null), null);
                     PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value"));
                     MemoryIndex memoryIndex = new MemoryIndex();
                     memoryIndex.addField("field", "value", new WhitespaceAnalyzer());
@@ -87,7 +87,7 @@ public void testHitsExecute() throws Exception {
 
                 // No match:
                 {
-                    HitContext hit = new HitContext(SearchHit.unpooled(0), context, 0, Map.of(), Source.empty(null));
+                    HitContext hit = new HitContext(SearchHit.unpooled(0), context, 0, Map.of(), Source.empty(null), null);
                     PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value"));
                     MemoryIndex memoryIndex = new MemoryIndex();
                     memoryIndex.addField("field", "value1", new WhitespaceAnalyzer());
@@ -117,7 +117,7 @@ public void testHitsExecute() throws Exception {
 
                 // No query:
                 {
-                    HitContext hit = new HitContext(SearchHit.unpooled(0), context, 0, Map.of(), Source.empty(null));
+                    HitContext hit = new HitContext(SearchHit.unpooled(0), context, 0, Map.of(), Source.empty(null), null);
                     PercolateQuery.QueryStore queryStore = ctx -> docId -> null;
                     MemoryIndex memoryIndex = new MemoryIndex();
                     memoryIndex.addField("field", "value", new WhitespaceAnalyzer());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java
index a4e2fda0fd3c9..dd002d3f44493 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/rank/FieldBasedRerankerIT.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.search.rank;
 
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
@@ -549,6 +550,11 @@ public boolean isCompoundBuilder() {
             return false;
         }
 
+        @Override
+        public Explanation explainHit(Explanation baseExplanation, RankDoc scoreDoc, List<String> queryNames) {
+            return baseExplanation;
+        }
+
         @Override
         public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
             return new QueryPhaseRankShardContext(queries, rankWindowSize()) {
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
index 72771855ff622..0d9844241d69d 100644
--- a/server/src/main/java/org/elasticsearch/TransportVersions.java
+++ b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -185,6 +185,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ADD_MISTRAL_EMBEDDINGS_INFERENCE = def(8_676_00_0);
     public static final TransportVersion ML_CHUNK_INFERENCE_OPTION = def(8_677_00_0);
     public static final TransportVersion RANK_FEATURE_PHASE_ADDED = def(8_678_00_0);
+    public static final TransportVersion RANK_DOC_IN_SHARD_FETCH_REQUEST = def(8_679_00_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 2308f5fcc8085..c81f5a20bc2d1 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -17,8 +17,13 @@
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.ShardSearchContextId;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.rank.RankDocShardInfo;
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.function.BiFunction;
 
 /**
@@ -117,6 +122,10 @@ private void innerRun() {
                     .forEach(searchPhaseShardResult -> releaseIrrelevantSearchContext(searchPhaseShardResult, context));
                 moveToNextPhase(fetchResults.getAtomicArray());
             } else {
+                final boolean shouldExplainRank = shouldExplainRankScores(context.getRequest());
+                final List<Map<Integer, RankDoc>> rankDocsPerShard = false == shouldExplainRank
+                    ? null
+                    : splitRankDocsPerShard(scoreDocs, numShards);
                 final ScoreDoc[] lastEmittedDocPerShard = context.getRequest().scroll() != null
                     ? SearchPhaseController.getLastEmittedDocPerShard(reducedQueryPhase, numShards)
                     : null;
@@ -129,6 +138,9 @@ private void innerRun() {
                 );
                 for (int i = 0; i < docIdsToLoad.length; i++) {
                     List<Integer> entry = docIdsToLoad[i];
+                    RankDocShardInfo rankDocs = rankDocsPerShard == null || rankDocsPerShard.get(i).isEmpty()
+                        ? null
+                        : new RankDocShardInfo(rankDocsPerShard.get(i));
                     SearchPhaseResult shardPhaseResult = searchPhaseShardResults.get(i);
                     if (entry == null) { // no results for this shard ID
                         if (shardPhaseResult != null) {
@@ -141,13 +153,34 @@ private void innerRun() {
                         // in any case we count down this result since we don't talk to this shard anymore
                         counter.countDown();
                     } else {
-                        executeFetch(shardPhaseResult, counter, entry, (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[i] : null);
+                        executeFetch(
+                            shardPhaseResult,
+                            counter,
+                            entry,
+                            rankDocs,
+                            (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[i] : null
+                        );
                     }
                 }
             }
         }
     }
 
+    private List<Map<Integer, RankDoc>> splitRankDocsPerShard(ScoreDoc[] scoreDocs, int numShards) {
+        List<Map<Integer, RankDoc>> rankDocsPerShard = new ArrayList<>(numShards);
+        for (int i = 0; i < numShards; i++) {
+            rankDocsPerShard.add(new HashMap<>());
+        }
+        for (ScoreDoc scoreDoc : scoreDocs) {
+            assert scoreDoc instanceof RankDoc : "ScoreDoc is not a RankDoc";
+            assert scoreDoc.shardIndex >= 0 && scoreDoc.shardIndex <= numShards;
+            RankDoc rankDoc = (RankDoc) scoreDoc;
+            Map<Integer, RankDoc> shardScoreDocs = rankDocsPerShard.get(rankDoc.shardIndex);
+            shardScoreDocs.put(rankDoc.doc, rankDoc);
+        }
+        return rankDocsPerShard;
+    }
+
     private boolean assertConsistentWithQueryAndFetchOptimization() {
         var phaseResults = searchPhaseShardResults.asList();
         assert phaseResults.isEmpty() || phaseResults.get(0).fetchResult() != null
@@ -159,6 +192,7 @@ private void executeFetch(
         SearchPhaseResult shardPhaseResult,
         final CountedCollector<FetchSearchResult> counter,
         final List<Integer> entry,
+        final RankDocShardInfo rankDocs,
         ScoreDoc lastEmittedDocForShard
     ) {
         final SearchShardTarget shardTarget = shardPhaseResult.getSearchShardTarget();
@@ -174,6 +208,7 @@ private void executeFetch(
                     contextId,
                     shardPhaseResult.getShardSearchRequest(),
                     entry,
+                    rankDocs,
                     lastEmittedDocForShard,
                     shardPhaseResult.getRescoreDocIds(),
                     aggregatedDfs
@@ -213,4 +248,12 @@ private void moveToNextPhase(AtomicArray<? extends SearchPhaseResult> fetchResul
         fetchResults.close();
         context.executeNextPhase(this, nextPhaseFactory.apply(resp, searchPhaseShardResults));
     }
+
+    private boolean shouldExplainRankScores(SearchRequest request) {
+        return request.source() != null
+            && request.source().explain() != null
+            && request.source().explain()
+            && request.source().rankBuilder() != null;
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 3e4f6dfec9fdb..6dddac290262c 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -437,9 +437,6 @@ public ActionRequestValidationException validate() {
                 if (source.profile()) {
                     validationException = addValidationError("[rank] requires [profile] is [false]", validationException);
                 }
-                if (source.explain() != null && source.explain()) {
-                    validationException = addValidationError("[rank] requires [explain] is [false]", validationException);
-                }
             }
             if (source.rescores() != null) {
                 for (@SuppressWarnings("rawtypes")
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index d93ff91a6ffe4..ef80ce706704a 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -226,6 +226,8 @@
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.PlainHighlighter;
 import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.rank.feature.RankFeatureDoc;
 import org.elasticsearch.search.rank.feature.RankFeatureShardPhase;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescorerBuilder;
@@ -332,6 +334,7 @@ public SearchModule(Settings settings, List<SearchPlugin> plugins, TelemetryProv
         registerRetrieverParsers(plugins);
         registerQueryParsers(plugins);
         registerRescorers(plugins);
+        registerRankers();
         registerSorts();
         registerValueFormats();
         registerSignificanceHeuristics(plugins);
@@ -828,6 +831,10 @@ private void registerRescorer(RescorerSpec<?> spec) {
         namedWriteables.add(new NamedWriteableRegistry.Entry(RescorerBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
     }
 
+    private void registerRankers() {
+        namedWriteables.add(new NamedWriteableRegistry.Entry(RankDoc.class, RankFeatureDoc.NAME, RankFeatureDoc::new));
+    }
+
     private void registerSorts() {
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, GeoDistanceSortBuilder.NAME, GeoDistanceSortBuilder::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScoreSortBuilder.NAME, ScoreSortBuilder::new));
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 3f9dd7895f6a7..b45a2e2e2ca14 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -733,7 +733,7 @@ public void executeRankFeaturePhase(RankFeatureShardRequest request, SearchShard
                     return searchContext.rankFeatureResult();
                 }
                 rankFeatureShardPhase.prepareForFetch(searchContext, request);
-                fetchPhase.execute(searchContext, docIds);
+                fetchPhase.execute(searchContext, docIds, null);
                 rankFeatureShardPhase.processFetch(searchContext);
                 var rankFeatureResult = searchContext.rankFeatureResult();
                 rankFeatureResult.incRef();
@@ -751,7 +751,7 @@ private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchCon
             Releasable scope = tracer.withScope(context.getTask());
             SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, afterQueryTime)
         ) {
-            fetchPhase.execute(context, shortcutDocIdsToLoad(context));
+            fetchPhase.execute(context, shortcutDocIdsToLoad(context), null);
             if (reader.singleSession()) {
                 freeReaderContext(reader.id());
             }
@@ -904,7 +904,7 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A
                 try (
                     SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext, true, System.nanoTime())
                 ) {
-                    fetchPhase.execute(searchContext, request.docIds());
+                    fetchPhase.execute(searchContext, request.docIds(), request.getRankDocks());
                     if (readerContext.singleSession()) {
                         freeReaderContext(request.contextId());
                     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregator.java
index 92fb09b017b2c..a3cf20d0b9b72 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregator.java
@@ -225,7 +225,7 @@ public SearchExecutionContext getSearchExecutionContext() {
                 return searchExecutionContext;
             }
         };
-        fetchSubSearchContext.fetchPhase().execute(fetchSubSearchContext, docIdsToLoad);
+        fetchSubSearchContext.fetchPhase().execute(fetchSubSearchContext, docIdsToLoad, null);
         return fetchSubSearchContext.fetchResult();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java
index 911c66f2fd533..65d49f771a045 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.rank.RankBuilder;
 import org.elasticsearch.search.rescore.RescoreContext;
 
 import java.util.Collections;
@@ -155,6 +156,19 @@ public List<RescoreContext> rescore() {
         return searchContext.rescore();
     }
 
+    /**
+     * The rank builder used in the original search
+     */
+    public RankBuilder rankBuilder() {
+        return searchContext.request().source() == null ? null : searchContext.request().source().rankBuilder();
+    }
+
+    public List<String> queryNames() {
+        return searchContext.request().source() == null
+            ? Collections.emptyList()
+            : searchContext.request().source().subSearches().stream().map(x -> x.getQueryBuilder().queryName()).toList();
+    }
+
     /**
      * Should the response include sequence number and primary term metadata
      */
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
index 4ba191794413d..db5617b543577 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -31,6 +31,8 @@
 import org.elasticsearch.search.profile.ProfileResult;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.profile.Timer;
+import org.elasticsearch.search.rank.RankDoc;
+import org.elasticsearch.search.rank.RankDocShardInfo;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.xcontent.XContentType;
 
@@ -56,7 +58,7 @@ public FetchPhase(List<FetchSubPhase> fetchSubPhases) {
         this.fetchSubPhases[fetchSubPhases.size()] = new InnerHitsPhase(this);
     }
 
-    public void execute(SearchContext context, int[] docIdsToLoad) {
+    public void execute(SearchContext context, int[] docIdsToLoad, RankDocShardInfo rankDocs) {
         if (LOGGER.isTraceEnabled()) {
             LOGGER.trace("{}", new SearchContextSourcePrinter(context));
         }
@@ -75,7 +77,7 @@ public void execute(SearchContext context, int[] docIdsToLoad) {
         Profiler profiler = context.getProfilers() == null ? Profiler.NOOP : Profilers.startProfilingFetchPhase();
         SearchHits hits = null;
         try {
-            hits = buildSearchHits(context, docIdsToLoad, profiler);
+            hits = buildSearchHits(context, docIdsToLoad, profiler, rankDocs);
         } finally {
             // Always finish profiling
             ProfileResult profileResult = profiler.finish();
@@ -97,7 +99,8 @@ public Source getSource(LeafReaderContext ctx, int doc) {
         }
     }
 
-    private SearchHits buildSearchHits(SearchContext context, int[] docIdsToLoad, Profiler profiler) {
+    private SearchHits buildSearchHits(SearchContext context, int[] docIdsToLoad, Profiler profiler, RankDocShardInfo rankDocs) {
+
         FetchContext fetchContext = new FetchContext(context);
         SourceLoader sourceLoader = context.newSourceLoader();
 
@@ -164,7 +167,8 @@ protected SearchHit nextDoc(int doc) throws IOException {
                     doc,
                     ctx,
                     leafSourceLoader,
-                    leafIdLoader
+                    leafIdLoader,
+                    rankDocs == null ? null : rankDocs.get(doc)
                 );
                 boolean success = false;
                 try {
@@ -221,7 +225,8 @@ private static HitContext prepareHitContext(
         int docId,
         LeafReaderContext subReaderContext,
         SourceLoader.Leaf sourceLoader,
-        IdLoader.Leaf idLoader
+        IdLoader.Leaf idLoader,
+        RankDoc rankDoc
     ) throws IOException {
         if (nestedDocuments.advance(docId - subReaderContext.docBase) == null) {
             return prepareNonNestedHitContext(
@@ -231,7 +236,8 @@ private static HitContext prepareHitContext(
                 docId,
                 subReaderContext,
                 sourceLoader,
-                idLoader
+                idLoader,
+                rankDoc
             );
         } else {
             return prepareNestedHitContext(
@@ -241,7 +247,8 @@ private static HitContext prepareHitContext(
                 docId,
                 nestedDocuments,
                 subReaderContext,
-                leafStoredFieldLoader
+                leafStoredFieldLoader,
+                rankDoc
             );
         }
     }
@@ -260,7 +267,8 @@ private static HitContext prepareNonNestedHitContext(
         int docId,
         LeafReaderContext subReaderContext,
         SourceLoader.Leaf sourceLoader,
-        IdLoader.Leaf idLoader
+        IdLoader.Leaf idLoader,
+        RankDoc rankDoc
     ) throws IOException {
         int subDocId = docId - subReaderContext.docBase;
 
@@ -271,7 +279,7 @@ private static HitContext prepareNonNestedHitContext(
             SearchHit hit = new SearchHit(docId);
             // TODO: can we use real pooled buffers here as well?
             Source source = Source.lazy(lazyStoredSourceLoader(profiler, subReaderContext, subDocId));
-            return new HitContext(hit, subReaderContext, subDocId, Map.of(), source);
+            return new HitContext(hit, subReaderContext, subDocId, Map.of(), source, rankDoc);
         } else {
             SearchHit hit = new SearchHit(docId, id);
             Source source;
@@ -287,7 +295,7 @@ private static HitContext prepareNonNestedHitContext(
             } else {
                 source = Source.lazy(lazyStoredSourceLoader(profiler, subReaderContext, subDocId));
             }
-            return new HitContext(hit, subReaderContext, subDocId, leafStoredFieldLoader.storedFields(), source);
+            return new HitContext(hit, subReaderContext, subDocId, leafStoredFieldLoader.storedFields(), source, rankDoc);
         }
     }
 
@@ -319,7 +327,8 @@ private static HitContext prepareNestedHitContext(
         int topDocId,
         LeafNestedDocuments nestedInfo,
         LeafReaderContext subReaderContext,
-        LeafStoredFieldLoader childFieldLoader
+        LeafStoredFieldLoader childFieldLoader,
+        RankDoc rankDoc
     ) throws IOException {
 
         String rootId;
@@ -351,7 +360,7 @@ private static HitContext prepareNestedHitContext(
         Source nestedSource = nestedIdentity.extractSource(rootSource);
 
         SearchHit hit = new SearchHit(topDocId, rootId, nestedIdentity);
-        return new HitContext(hit, subReaderContext, nestedInfo.doc(), childFieldLoader.storedFields(), nestedSource);
+        return new HitContext(hit, subReaderContext, nestedInfo.doc(), childFieldLoader.storedFields(), nestedSource, rankDoc);
     }
 
     interface Profiler {
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
index a6f41f8b7fed3..d6de6d46462e4 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
@@ -11,8 +11,10 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.ReaderUtil;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.rank.RankDoc;
 
 import java.io.IOException;
 import java.util.List;
@@ -29,13 +31,22 @@ class HitContext {
         private final int docId;
         private final Source source;
         private final Map<String, List<Object>> loadedFields;
+        private final RankDoc rankDoc;
 
-        public HitContext(SearchHit hit, LeafReaderContext context, int docId, Map<String, List<Object>> loadedFields, Source source) {
+        public HitContext(
+            SearchHit hit,
+            LeafReaderContext context,
+            int docId,
+            Map<String, List<Object>> loadedFields,
+            Source source,
+            RankDoc rankDoc
+        ) {
             this.hit = hit;
             this.readerContext = context;
             this.docId = docId;
             this.source = source;
             this.loadedFields = loadedFields;
+            this.rankDoc = rankDoc;
         }
 
         public SearchHit hit() {
@@ -72,6 +83,11 @@ public Map<String, List<Object>> loadedFields() {
             return loadedFields;
         }
 
+        @Nullable
+        public RankDoc rankDoc() {
+            return this.rankDoc;
+        }
+
         public IndexReader topLevelReader() {
             return ReaderUtil.getTopLevelContext(readerContext).reader();
         }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
index 86f6db0b681d7..8128f48dda013 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchRequest.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.search.rank.RankDocShardInfo;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportRequest;
@@ -118,4 +119,9 @@ public RescoreDocIds getRescoreDocIds() {
     public AggregatedDfs getAggregatedDfs() {
         return null;
     }
+
+    @Nullable
+    public RankDocShardInfo getRankDocks() {
+        return null;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
index a0f960dc4aaad..0415ecc4a6498 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.search.fetch;
 
 import org.apache.lucene.search.ScoreDoc;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -18,6 +19,7 @@
 import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.search.rank.RankDocShardInfo;
 
 import java.io.IOException;
 import java.util.List;
@@ -32,12 +34,14 @@ public class ShardFetchSearchRequest extends ShardFetchRequest implements Indice
     private final ShardSearchRequest shardSearchRequest;
     private final RescoreDocIds rescoreDocIds;
     private final AggregatedDfs aggregatedDfs;
+    private final RankDocShardInfo rankDocs;
 
     public ShardFetchSearchRequest(
         OriginalIndices originalIndices,
         ShardSearchContextId id,
         ShardSearchRequest shardSearchRequest,
         List<Integer> docIds,
+        RankDocShardInfo rankDocs,
         ScoreDoc lastEmittedDoc,
         RescoreDocIds rescoreDocIds,
         AggregatedDfs aggregatedDfs
@@ -47,6 +51,7 @@ public ShardFetchSearchRequest(
         this.shardSearchRequest = shardSearchRequest;
         this.rescoreDocIds = rescoreDocIds;
         this.aggregatedDfs = aggregatedDfs;
+        this.rankDocs = rankDocs;
     }
 
     public ShardFetchSearchRequest(StreamInput in) throws IOException {
@@ -55,6 +60,11 @@ public ShardFetchSearchRequest(StreamInput in) throws IOException {
         shardSearchRequest = in.readOptionalWriteable(ShardSearchRequest::new);
         rescoreDocIds = new RescoreDocIds(in);
         aggregatedDfs = in.readOptionalWriteable(AggregatedDfs::new);
+        if (in.getTransportVersion().onOrAfter(TransportVersions.RANK_DOC_IN_SHARD_FETCH_REQUEST)) {
+            this.rankDocs = in.readOptionalWriteable(RankDocShardInfo::new);
+        } else {
+            this.rankDocs = null;
+        }
     }
 
     @Override
@@ -64,6 +74,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(shardSearchRequest);
         rescoreDocIds.writeTo(out);
         out.writeOptionalWriteable(aggregatedDfs);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.RANK_DOC_IN_SHARD_FETCH_REQUEST)) {
+            out.writeOptionalWriteable(rankDocs);
+        }
     }
 
     @Override
@@ -96,4 +109,9 @@ public RescoreDocIds getRescoreDocIds() {
     public AggregatedDfs getAggregatedDfs() {
         return aggregatedDfs;
     }
+
+    @Override
+    public RankDocShardInfo getRankDocks() {
+        return this.rankDocs;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java
index 0873ca777d428..16f07b2ab9880 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.search.rescore.RescoreContext;
 
 import java.io.IOException;
+import java.util.List;
 
 /**
  * Explains the scoring calculations for the top hits.
@@ -27,6 +28,9 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) {
             return null;
         }
         return new FetchSubPhaseProcessor() {
+
+            private final List<String> queryNames = context.queryNames();
+
             @Override
             public void setNextReader(LeafReaderContext readerContext) {
 
@@ -40,6 +44,9 @@ public void process(HitContext hitContext) throws IOException {
                 for (RescoreContext rescore : context.rescore()) {
                     explanation = rescore.rescorer().explain(topLevelDocId, context.searcher(), rescore, explanation);
                 }
+                if (context.rankBuilder() != null) {
+                    explanation = context.rankBuilder().explainHit(explanation, hitContext.rankDoc(), queryNames);
+                }
                 // we use the top level doc id, since we work with the top level searcher
                 hitContext.hit().explanation(explanation);
             }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java
index a4ba982e1dd73..61e3b15d530f7 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java
@@ -92,7 +92,7 @@ private void hitExecute(Map<String, InnerHitsContext.InnerHitSubContext> innerHi
             innerHitsContext.setRootId(hit.getId());
             innerHitsContext.setRootLookup(rootSource);
 
-            fetchPhase.execute(innerHitsContext, docIdsToLoad);
+            fetchPhase.execute(innerHitsContext, docIdsToLoad, null);
             FetchSearchResult fetchResult = innerHitsContext.fetchResult();
             SearchHit[] internalHits = fetchResult.fetchResult().hits().getHits();
             for (int j = 0; j < internalHits.length; j++) {
diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java b/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java
index f496758c3f5c6..704ead39b1ded 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/RankBuilder.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.search.rank;
 
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -76,6 +77,13 @@ public int rankWindowSize() {
      */
     public abstract boolean isCompoundBuilder();
 
+    /**
+     * Generates an {@code Explanation} on how the final score for the provided {@code RankDoc} is computed for the given `RankBuilder`.
+     * In addition to the base explanation to enrich, we also have access to the query names that were provided in the request,
+     * so that we can have direct association with the user provided query.
+     */
+    public abstract Explanation explainHit(Explanation baseExplanation, RankDoc scoreDoc, List<String> queryNames);
+
     /**
      * Generates a context used to execute required searches during the query phase on the shard.
      */
diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java b/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java
index bd177008dd902..50b3ddc0f370a 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java
@@ -9,9 +9,9 @@
 package org.elasticsearch.search.rank;
 
 import org.apache.lucene.search.ScoreDoc;
+import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -21,7 +21,7 @@
  * Subclasses should extend this with additional information
  * required for their global ranking method.
  */
-public abstract class RankDoc extends ScoreDoc implements Writeable {
+public abstract class RankDoc extends ScoreDoc implements NamedWriteable {
 
     public static final int NO_RANK = -1;
 
@@ -37,7 +37,7 @@ public RankDoc(int doc, float score, int shardIndex) {
         super(doc, score, shardIndex);
     }
 
-    protected RankDoc(StreamInput in) throws IOException {
+    public RankDoc(StreamInput in) throws IOException {
         super(in.readVInt(), in.readFloat(), in.readVInt());
         rank = in.readVInt();
     }
diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankDocShardInfo.java b/server/src/main/java/org/elasticsearch/search/rank/RankDocShardInfo.java
new file mode 100644
index 0000000000000..56866dba36159
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/rank/RankDocShardInfo.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.rank;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * A {@code RankDocShardInfo} holds all the final rank documents that exist in a shard. We pass this
+ * to fetchPhase so that we can pass all the needed information for the RankBuilder to perform any actions needed
+ * when building the final SearchHits (e.g. explain).
+ */
+public class RankDocShardInfo implements Writeable {
+
+    // doc-id to RankDoc mapping
+    private final Map<Integer, RankDoc> rankDocs;
+
+    public RankDocShardInfo(Map<Integer, RankDoc> rankDocs) {
+        this.rankDocs = rankDocs;
+    }
+
+    public RankDocShardInfo(StreamInput in) throws IOException {
+        rankDocs = in.readMap(StreamInput::readVInt, v -> v.readNamedWriteable(RankDoc.class));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(rankDocs, StreamOutput::writeVInt, StreamOutput::writeNamedWriteable);
+    }
+
+    public RankDoc get(int index) {
+        return rankDocs.get(index);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java
index 8eb3f2fc8339b..d8b4ec10410f1 100644
--- a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java
+++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java
@@ -20,6 +20,8 @@
  */
 public class RankFeatureDoc extends RankDoc {
 
+    public static final String NAME = "rank_feature_doc";
+
     // todo: update to support more than 1 fields; and not restrict to string data
     public String featureData;
 
@@ -51,4 +53,9 @@ protected boolean doEquals(RankDoc rd) {
     protected int doHashCode() {
         return Objects.hashCode(featureData);
     }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java
index 3c4355e56d21d..b369324b3ee52 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java
@@ -133,6 +133,9 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder
         if (preFilterQueryBuilders != null) {
             knnSearchBuilder.addFilterQueries(preFilterQueryBuilders);
         }
+        if (retrieverName != null) {
+            knnSearchBuilder.queryName(retrieverName);
+        }
         List<KnnSearchBuilder> knnSearchBuilders = new ArrayList<>(searchSourceBuilder.knnSearch());
         knnSearchBuilders.add(knnSearchBuilder);
         searchSourceBuilder.knnSearch(knnSearchBuilders);
diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
index c9b12f03beb53..6e3d2a58dbd5d 100644
--- a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java
@@ -48,6 +48,8 @@ public abstract class RetrieverBuilder implements ToXContent {
 
     public static final ParseField PRE_FILTER_FIELD = new ParseField("filter");
 
+    public static final ParseField NAME_FIELD = new ParseField("_name");
+
     protected static void declareBaseParserFields(
         String name,
         AbstractObjectParser<? extends RetrieverBuilder, RetrieverParserContext> parser
@@ -57,6 +59,11 @@ protected static void declareBaseParserFields(
             c.trackSectionUsage(name + ":" + PRE_FILTER_FIELD.getPreferredName());
             return preFilterQueryBuilder;
         }, PRE_FILTER_FIELD);
+        parser.declareString(RetrieverBuilder::retrieverName, NAME_FIELD);
+    }
+
+    private void retrieverName(String retrieverName) {
+        this.retrieverName = retrieverName;
     }
 
     /**
@@ -172,6 +179,8 @@ protected static RetrieverBuilder parseInnerRetrieverBuilder(XContentParser pars
 
     protected List<QueryBuilder> preFilterQueryBuilders = new ArrayList<>();
 
+    protected String retrieverName;
+
     /**
      * Gets the filters for this retriever.
      */
diff --git a/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java
index 9716749562eae..836e88b3a12ef 100644
--- a/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.action.search;
 
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
@@ -32,6 +33,7 @@
 import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rank.RankBuilder;
+import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.rank.RankShardResult;
 import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
@@ -992,6 +994,12 @@ public boolean isCompoundBuilder() {
                 return true;
             }
 
+            @Override
+            public Explanation explainHit(Explanation baseExplanation, RankDoc scoreDoc, List<String> queryNames) {
+                // no-op
+                return baseExplanation;
+            }
+
             @Override
             public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
                 return queryPhaseRankShardContext;
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
index d8c7d3e134571..cd176af3dafad 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
@@ -529,18 +529,6 @@ public void testValidate() throws IOException {
             assertEquals(1, validationErrors.validationErrors().size());
             assertEquals("[rank] requires [profile] is [false]", validationErrors.validationErrors().get(0));
         }
-        {
-            SearchRequest searchRequest = new SearchRequest().source(
-                new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100))
-                    .query(QueryBuilders.termQuery("field", "term"))
-                    .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null)))
-                    .explain(true)
-            );
-            ActionRequestValidationException validationErrors = searchRequest.validate();
-            assertNotNull(validationErrors);
-            assertEquals(1, validationErrors.validationErrors().size());
-            assertEquals("[rank] requires [explain] is [false]", validationErrors.validationErrors().get(0));
-        }
         {
             SearchRequest searchRequest = new SearchRequest("test").source(
                 new SearchSourceBuilder().pointInTimeBuilder(new PointInTimeBuilder(BytesArray.EMPTY))
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 2af20a6ffef4a..5c175f792d399 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -587,6 +587,7 @@ public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId)
                 request,
                 globalTopKResults,
                 null,
+                null,
                 rankResult.getRescoreDocIds(),
                 null
             );
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java
index 3a7460c05ca87..a086225e140ac 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java
@@ -88,7 +88,7 @@ public void testDocValueFetcher() throws IOException {
             processor.setNextReader(context);
             for (int doc = 0; doc < context.reader().maxDoc(); doc++) {
                 SearchHit searchHit = SearchHit.unpooled(doc + context.docBase);
-                processor.process(new FetchSubPhase.HitContext(searchHit, context, doc, Map.of(), Source.empty(null)));
+                processor.process(new FetchSubPhase.HitContext(searchHit, context, doc, Map.of(), Source.empty(null), null));
                 assertNotNull(searchHit.getFields().get("field"));
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java
index 2b8bf0dad65fe..8778d02dc44f9 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java
@@ -197,7 +197,7 @@ private HitContext hitExecuteMultiple(
         MemoryIndex index = new MemoryIndex();
         LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0);
         Source source = sourceBuilder == null ? Source.empty(null) : Source.fromBytes(BytesReference.bytes(sourceBuilder));
-        HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1, Map.of(), source);
+        HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1, Map.of(), source, null);
 
         FetchSourcePhase phase = new FetchSourcePhase();
         FetchSubPhaseProcessor processor = phase.getProcessor(fetchContext);
diff --git a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
index a8425a1c59d9a..9a6442e129d5a 100644
--- a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.search.rank;
 
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.TransportVersion;
@@ -129,6 +130,12 @@ public boolean isCompoundBuilder() {
                 return false;
             }
 
+            @Override
+            public Explanation explainHit(Explanation baseExplanation, RankDoc scoreDoc, List<String> queryNames) {
+                // no-op
+                return baseExplanation;
+            }
+
             // no work to be done on the query phase
             @Override
             public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
diff --git a/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java
index 526c2104e52ae..a6c76bc15119c 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java
@@ -77,7 +77,8 @@ protected final Map<String, HighlightField> highlight(MapperService mapperServic
                 ir.leaves().get(0),
                 0,
                 storedFields,
-                source
+                source,
+                null
             );
             processor.process(hitContext);
             highlights.putAll(hitContext.hit().getHighlightFields());
diff --git a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java
index 862c4d2ea3270..5694b1a2592ef 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankBuilder.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.search.rank;
 
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
@@ -81,6 +82,11 @@ public boolean isCompoundBuilder() {
         return true;
     }
 
+    @Override
+    public Explanation explainHit(Explanation baseExplanation, RankDoc rankDoc, List<String> queryNames) {
+        return baseExplanation;
+    }
+
     @Override
     public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
         throw new UnsupportedOperationException();
diff --git a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankDoc.java b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankDoc.java
index 0a8b6e4c5f2be..f2f3cb82d203f 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankDoc.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/rank/TestRankDoc.java
@@ -37,4 +37,9 @@ public boolean doEquals(RankDoc rd) {
     public int doHashCode() {
         return 0;
     }
+
+    @Override
+    public String getWriteableName() {
+        return "test_rank_doc";
+    }
 }
diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java
index af465658a0b52..b501967524a6b 100644
--- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java
+++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankMultiShardIT.java
@@ -29,7 +29,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 
-@ESIntegTestCase.ClusterScope(maxNumDataNodes = 3)
+@ESIntegTestCase.ClusterScope(minNumDataNodes = 2, maxNumDataNodes = 4)
 @ESIntegTestCase.SuiteScopeTestCase
 public class RRFRankMultiShardIT extends ESIntegTestCase {
 
@@ -973,4 +973,234 @@ public void testMultiBM25AndMultipleKnnWithAggregation() {
             }
         );
     }
+
+    public void testBasicRRFExplain() {
+        // our query here is a top-level knn query for vector [9] and a term query for "text0: 10"
+        // the first result should be the one present in both queries (i.e. doc with text0: 10 and vector: [10]) and the other ones
+        // should only match the knn query
+        float[] queryVector = { 9f };
+        KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null).queryName("my_knn_search");
+        assertResponse(
+            prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(100, 1))
+                .setKnnSearch(List.of(knnSearch))
+                .setQuery(QueryBuilders.termQuery("text0", "10"))
+                .setExplain(true)
+                .setSize(3),
+            response -> {
+                // we cast to Number when looking at values in vector fields because different xContentTypes may return Float or Double
+                assertEquals(3, response.getHits().getHits().length);
+
+                // first result is the one which matches the term (10) so we should expect an explanation for both queries
+                SearchHit hit = response.getHits().getAt(0);
+                assertEquals(1, hit.getRank());
+                assertTrue(hit.getExplanation().isMatch());
+                assertTrue(hit.getExplanation().getDescription().contains("initial ranks"));
+                assertEquals(2, hit.getExplanation().getDetails().length);
+                assertTrue(hit.getExplanation().getDetails()[0].isMatch());
+                assertEquals(1, hit.getExplanation().getDetails()[0].getValue().intValue());
+                assertTrue(hit.getExplanation().getDetails()[0].getDescription().contains("query at index [0]"));
+                assertTrue(hit.getExplanation().getDetails()[0].getDetails().length > 0);
+                assertTrue(hit.getExplanation().getDetails()[1].isMatch());
+                assertTrue(hit.getExplanation().getDetails()[1].getDescription().contains("[my_knn_search]"));
+                assertTrue(hit.getExplanation().getDetails()[1].getDetails().length > 0);
+
+                // second result matched only on the knn query so no match should be expected for the term query
+                hit = response.getHits().getAt(1);
+                assertEquals(2, hit.getRank());
+                assertTrue(hit.getExplanation().isMatch());
+                assertTrue(hit.getExplanation().getDescription().contains("initial ranks"));
+                assertEquals(2, hit.getExplanation().getDetails().length);
+                assertFalse(hit.getExplanation().getDetails()[0].isMatch());
+                assertEquals(0, hit.getExplanation().getDetails()[0].getValue().intValue());
+                assertEquals(
+                    "rrf score: [0], result not found in query at index [0]",
+                    hit.getExplanation().getDetails()[0].getDescription()
+                );
+                assertEquals(0, hit.getExplanation().getDetails()[0].getDetails().length);
+                assertTrue(hit.getExplanation().getDetails()[1].isMatch());
+                assertTrue(hit.getExplanation().getDetails()[1].getDescription().contains("[my_knn_search]"));
+                assertTrue(hit.getExplanation().getDetails()[1].getDetails().length > 0);
+
+                // third result matched only on the knn query so no match should be expected for the term query
+                hit = response.getHits().getAt(2);
+                assertEquals(3, hit.getRank());
+                assertTrue(hit.getExplanation().isMatch());
+                assertTrue(hit.getExplanation().getDescription().contains("initial ranks"));
+                assertEquals(2, hit.getExplanation().getDetails().length);
+                assertFalse(hit.getExplanation().getDetails()[0].isMatch());
+                assertEquals(0, hit.getExplanation().getDetails()[0].getValue().intValue());
+                assertEquals(
+                    "rrf score: [0], result not found in query at index [0]",
+                    hit.getExplanation().getDetails()[0].getDescription()
+                );
+                assertEquals(0, hit.getExplanation().getDetails()[0].getDetails().length);
+                assertTrue(hit.getExplanation().getDetails()[1].isMatch());
+                assertTrue(hit.getExplanation().getDetails()[1].getDescription().contains("[my_knn_search]"));
+                assertTrue(hit.getExplanation().getDetails()[1].getDetails().length > 0);
+            }
+        );
+    }
+
+    public void testRRFExplainUnknownField() {
+        // in this test we try knn with a query on an unknown field that would be rewritten to MatchNoneQuery
+        // so we expect results and explanations only for the first part
+        float[] queryVector = { 9f };
+        KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null).queryName("my_knn_search");
+        assertResponse(
+            prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(100, 1))
+                .setKnnSearch(List.of(knnSearch))
+                .setQuery(QueryBuilders.termQuery("unknown_field", "10"))
+                .setExplain(true)
+                .setSize(3),
+            response -> {
+                // we cast to Number when looking at values in vector fields because different xContentTypes may return Float or Double
+                assertEquals(3, response.getHits().getHits().length);
+
+                SearchHit hit = response.getHits().getAt(0);
+                assertEquals(1, hit.getRank());
+                assertTrue(hit.getExplanation().isMatch());
+                assertTrue(hit.getExplanation().getDescription().contains("initial ranks"));
+                assertEquals(2, hit.getExplanation().getDetails().length);
+                assertFalse(hit.getExplanation().getDetails()[0].isMatch());
+                assertEquals(0, hit.getExplanation().getDetails()[0].getValue().intValue());
+                assertEquals(
+                    "rrf score: [0], result not found in query at index [0]",
+                    hit.getExplanation().getDetails()[0].getDescription()
+                );
+                assertEquals(0, hit.getExplanation().getDetails()[0].getDetails().length);
+                assertTrue(hit.getExplanation().getDetails()[1].isMatch());
+                assertTrue(hit.getExplanation().getDetails()[1].getDescription().contains("[my_knn_search]"));
+                assertTrue(hit.getExplanation().getDetails()[1].getDetails().length > 0);
+
+                hit = response.getHits().getAt(1);
+                assertEquals(2, hit.getRank());
+                assertTrue(hit.getExplanation().isMatch());
+                assertTrue(hit.getExplanation().getDescription().contains("initial ranks"));
+                assertEquals(2, hit.getExplanation().getDetails().length);
+                assertFalse(hit.getExplanation().getDetails()[0].isMatch());
+                assertEquals(0, hit.getExplanation().getDetails()[0].getValue().intValue());
+                assertEquals(
+                    "rrf score: [0], result not found in query at index [0]",
+                    hit.getExplanation().getDetails()[0].getDescription()
+                );
+                assertEquals(0, hit.getExplanation().getDetails()[0].getDetails().length);
+                assertTrue(hit.getExplanation().getDetails()[1].isMatch());
+                assertTrue(hit.getExplanation().getDetails()[1].getDescription().contains("[my_knn_search]"));
+                assertTrue(hit.getExplanation().getDetails()[1].getDetails().length > 0);
+
+                hit = response.getHits().getAt(2);
+                assertEquals(3, hit.getRank());
+                assertTrue(hit.getExplanation().isMatch());
+                assertTrue(hit.getExplanation().getDescription().contains("initial ranks"));
+                assertEquals(2, hit.getExplanation().getDetails().length);
+                assertFalse(hit.getExplanation().getDetails()[0].isMatch());
+                assertEquals(0, hit.getExplanation().getDetails()[0].getValue().intValue());
+                assertEquals(
+                    "rrf score: [0], result not found in query at index [0]",
+                    hit.getExplanation().getDetails()[0].getDescription()
+                );
+                assertEquals(0, hit.getExplanation().getDetails()[0].getDetails().length, 0);
+                assertTrue(hit.getExplanation().getDetails()[1].isMatch());
+                assertTrue(hit.getExplanation().getDetails()[1].getDescription().contains("[my_knn_search]"));
+                assertTrue(hit.getExplanation().getDetails()[1].getDetails().length > 0);
+            }
+        );
+    }
+
+    public void testRRFExplainOneUnknownFieldSubSearches() {
+        // this test is similar to the above with the difference that we have a list of subsearches that one would fail,
+        // while the other one would produce a match.
+        // So, we'd have a total of 3 queries, a (rewritten) MatchNoneQuery, a TermQuery, and a kNN query
+        float[] queryVector = { 9f };
+        KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector_asc", queryVector, 101, 1001, null).queryName("my_knn_search");
+        assertResponse(
+            prepareSearch("nrd_index").setRankBuilder(new RRFRankBuilder(100, 1))
+                .setKnnSearch(List.of(knnSearch))
+                .setSubSearches(
+                    List.of(
+                        new SubSearchSourceBuilder(QueryBuilders.termQuery("unknown_field", "10")),
+                        new SubSearchSourceBuilder(QueryBuilders.termQuery("text0", "10"))
+                    )
+                )
+                .setExplain(true)
+                .setSize(3),
+            response -> {
+                // we cast to Number when looking at values in vector fields because different xContentTypes may return Float or Double
+                assertEquals(3, response.getHits().getHits().length);
+
+                // first result is the one which matches the term (10) and is 3rd closest to our query vector (9)
+                SearchHit hit = response.getHits().getAt(0);
+                assertEquals(1, hit.getRank());
+                assertTrue(hit.getExplanation().isMatch());
+                assertTrue(hit.getExplanation().getDescription().contains("initial ranks"));
+                assertEquals(3, hit.getExplanation().getDetails().length);
+                // MatchNone query
+                assertFalse(hit.getExplanation().getDetails()[0].isMatch());
+                assertEquals(0, hit.getExplanation().getDetails()[0].getValue().intValue());
+                assertEquals(
+                    "rrf score: [0], result not found in query at index [0]",
+                    hit.getExplanation().getDetails()[0].getDescription()
+                );
+                assertEquals(0, hit.getExplanation().getDetails()[0].getDetails().length);
+                // Term query
+                assertTrue(hit.getExplanation().getDetails()[1].isMatch());
+                assertTrue(hit.getExplanation().getDetails()[1].getDescription().contains("query at index [1]"));
+                assertTrue(hit.getExplanation().getDetails()[1].getDetails().length > 0);
+                // knn query
+                assertTrue(hit.getExplanation().getDetails()[2].isMatch());
+                assertTrue(hit.getExplanation().getDetails()[2].getDescription().contains("[my_knn_search]"));
+                assertTrue(hit.getExplanation().getDetails()[2].getDetails().length > 0);
+
+                // rest of hits match only on the knn query so no match should be expected for the term query either
+                hit = response.getHits().getAt(1);
+                assertEquals(2, hit.getRank());
+                assertTrue(hit.getExplanation().isMatch());
+                assertTrue(hit.getExplanation().getDescription().contains("initial ranks"));
+                assertEquals(3, hit.getExplanation().getDetails().length);
+                // MatchNone query
+                assertFalse(hit.getExplanation().getDetails()[0].isMatch());
+                assertEquals(0, hit.getExplanation().getDetails()[0].getValue().intValue());
+                assertEquals(
+                    "rrf score: [0], result not found in query at index [0]",
+                    hit.getExplanation().getDetails()[0].getDescription()
+                );
+                // term query - should not match
+                assertFalse(hit.getExplanation().getDetails()[1].isMatch());
+                assertEquals(
+                    "rrf score: [0], result not found in query at index [1]",
+                    hit.getExplanation().getDetails()[1].getDescription()
+                );
+                assertEquals(0, hit.getExplanation().getDetails()[1].getDetails().length);
+                // knn query
+                assertTrue(hit.getExplanation().getDetails()[2].isMatch());
+                assertTrue(hit.getExplanation().getDetails()[2].getDescription().contains("[my_knn_search]"));
+                assertTrue(hit.getExplanation().getDetails()[2].getDetails().length > 0);
+
+                // rest of hits match only on the knn query so no match should be expected for the term query either
+                hit = response.getHits().getAt(2);
+                assertEquals(3, hit.getRank());
+                assertTrue(hit.getExplanation().isMatch());
+                assertTrue(hit.getExplanation().getDescription().contains("initial ranks"));
+                assertEquals(3, hit.getExplanation().getDetails().length);
+                // MatchNone query
+                assertFalse(hit.getExplanation().getDetails()[0].isMatch());
+                assertEquals(0, hit.getExplanation().getDetails()[0].getValue().intValue());
+                assertEquals(
+                    "rrf score: [0], result not found in query at index [0]",
+                    hit.getExplanation().getDetails()[0].getDescription()
+                );
+                // term query - should not match
+                assertFalse(hit.getExplanation().getDetails()[1].isMatch());
+                assertEquals(
+                    "rrf score: [0], result not found in query at index [1]",
+                    hit.getExplanation().getDetails()[1].getDescription()
+                );
+                assertEquals(0, hit.getExplanation().getDetails()[1].getDetails().length);
+                // knn query
+                assertTrue(hit.getExplanation().getDetails()[2].isMatch());
+                assertTrue(hit.getExplanation().getDetails()[2].getDescription().contains("[my_knn_search]"));
+                assertTrue(hit.getExplanation().getDetails()[2].getDetails().length > 0);
+            }
+        );
+    }
 }
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
index 5c39c6c32fd06..dfd47f122cb61 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.rank.rrf;
 
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
@@ -14,6 +15,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.search.rank.RankBuilder;
+import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
 import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext;
 import org.elasticsearch.search.rank.context.RankFeaturePhaseRankCoordinatorContext;
@@ -25,6 +27,7 @@
 import org.elasticsearch.xpack.core.XPackPlugin;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 
@@ -101,6 +104,62 @@ public boolean isCompoundBuilder() {
         return true;
     }
 
+    @Override
+    public Explanation explainHit(Explanation baseExplanation, RankDoc rankDoc, List<String> queryNames) {
+        if (rankDoc == null) {
+            return baseExplanation;
+        }
+        if (false == baseExplanation.isMatch()) {
+            return baseExplanation;
+        }
+        final Explanation hitExplanation = baseExplanation.getDetails().length == 0
+            ? Explanation.match(baseExplanation.getValue(), baseExplanation.getDescription(), baseExplanation)
+            : baseExplanation;
+
+        assert rankDoc instanceof RRFRankDoc : "ScoreDoc is not an instance of RRFRankDoc";
+        RRFRankDoc rrfRankDoc = (RRFRankDoc) rankDoc;
+        int queries = rrfRankDoc.positions.length;
+        assert queryNames.size() == queries;
+        Explanation[] details = new Explanation[queries];
+        int queryExplainIndex = 0;
+        for (int i = 0; i < queries; i++) {
+            final String queryName = queryNames.get(i) != null ? "[" + queryNames.get(i) + "]" : "at index [" + i + "]";
+            if (rrfRankDoc.positions[i] == RRFRankDoc.NO_RANK) {
+                final String description = "rrf score: [0], result not found in query " + queryName;
+                details[i] = Explanation.noMatch(description);
+            } else {
+                final int rank = rrfRankDoc.positions[i] + 1;
+                details[i] = Explanation.match(
+                    rank,
+                    "rrf score: ["
+                        + (1f / (rank + rankConstant))
+                        + "], "
+                        + "for rank ["
+                        + (rank)
+                        + "] in query "
+                        + queryName
+                        + " computed as [1 / ("
+                        + (rank)
+                        + " + "
+                        + rankConstant
+                        + "]), for matching query with score: ",
+                    hitExplanation.getDetails()[queryExplainIndex++]
+                );
+            }
+        }
+        return Explanation.match(
+            rrfRankDoc.score,
+            "rrf score: ["
+                + rrfRankDoc.score
+                + "] computed for initial ranks "
+                + Arrays.toString(Arrays.stream(rrfRankDoc.positions).map(x -> x + 1).toArray())
+                + " with rankConstant: ["
+                + rankConstant
+                + "] as sum of [1 / (rank + rankConstant)] for each query",
+            details
+        );
+    }
+
     public QueryPhaseRankShardContext buildQueryPhaseShardContext(List<Query> queries, int from) {
         return new RRFQueryPhaseRankShardContext(queries, rankWindowSize(), rankConstant);
     }
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java
index 37bac43f827cf..8f078c0c4d116 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java
@@ -20,6 +20,8 @@
  */
 public class RRFRankDoc extends RankDoc {
 
+    static final String NAME = "rrf_rank_doc";
+
     /**
      * The position within each result set per query. The length
      * of {@code positions} is the number of queries that are part
@@ -89,4 +91,9 @@ public String toString() {
             + shardIndex
             + '}';
     }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
 }
diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java
index 4d7c60f00ec1c..ece08d1a3d558 100644
--- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java
+++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.rank.RankBuilder;
+import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.rank.RankShardResult;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
@@ -33,7 +34,8 @@ public class RRFRankPlugin extends Plugin implements SearchPlugin {
     public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         return List.of(
             new NamedWriteableRegistry.Entry(RankBuilder.class, NAME, RRFRankBuilder::new),
-            new NamedWriteableRegistry.Entry(RankShardResult.class, NAME, RRFRankShardResult::new)
+            new NamedWriteableRegistry.Entry(RankShardResult.class, NAME, RRFRankShardResult::new),
+            new NamedWriteableRegistry.Entry(RankDoc.class, RRFRankDoc.NAME, RRFRankDoc::new)
         );
     }
 
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/500_rrf_retriever_explain.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/500_rrf_retriever_explain.yml
new file mode 100644
index 0000000000000..8d74ecbccd328
--- /dev/null
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/500_rrf_retriever_explain.yml
@@ -0,0 +1,275 @@
+setup:
+  - requires:
+      cluster_features: "gte_v8.15.0"
+      reason: 'explain for rrf was added in 8.15'
+      test_runner_features: close_to
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              text:
+                type: text
+              integer:
+                type: integer
+              vector:
+                type: dense_vector
+                dims: 1
+                index: true
+                similarity: l2_norm
+                index_options:
+                  type: hnsw
+                  ef_construction: 100
+                  m: 16
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          text: "term"
+          integer: 1
+          vector: [5]
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          text: "term term"
+          integer: 2
+          vector: [4]
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          text: "term term term"
+          integer: 3
+          vector: [3]
+  - do:
+      index:
+        index: test
+        id: "4"
+        body:
+          text: "term term term term"
+          integer: 3
+
+  - do:
+      index:
+        index: test
+        id: "5"
+        body:
+          integer: 1
+          vector: [0]
+
+  - do:
+      indices.refresh: {}
+
+---
+"using rrf retriever":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          retriever:
+            rrf:
+              retrievers: [
+                {
+                  standard: {
+                    query: {
+                      term: {
+                        text: "term"
+                      }
+                    }
+                  }
+                },
+                {
+                  knn: {
+                      field: "vector",
+                      query_vector: [ 3 ],
+                      num_candidates: 5,
+                      k: 5
+                  }
+                }
+              ]
+              rank_window_size: 5
+              rank_constant: 1
+          size: 3
+          explain: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - close_to: { hits.hits.0._explanation.value: { value: 0.8333334, error: 0.000001 } }
+  - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.8333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.value: 2}
+  - match: {hits.hits.0._explanation.details.0.description: "/rrf.score:.\\[0.33333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.0._explanation.details.1.value: 1}
+  - match: {hits.hits.0._explanation.details.1.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } }
+  - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.value: 3}
+  - match: {hits.hits.1._explanation.details.0.description: "/rrf.score:.\\[0.25\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.1._explanation.details.1.value: 2}
+  - match: {hits.hits.1._explanation.details.1.description: "/rrf.score:.\\[0.33333334\\].*/" }
+  - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - match: {hits.hits.2._explanation.value: 0.5}
+  - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.value: 1}
+  - match: {hits.hits.2._explanation.details.0.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.2._explanation.details.1.value: 0}
+  - match: {hits.hits.2._explanation.details.1.description: "/rrf.score:.\\[0\\],.result.not.found./"}
+  - length: {hits.hits.2._explanation.details.1.details: 0}
+
+---
+"using named retrievers":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          retriever:
+            rrf:
+              retrievers: [
+                {
+                  standard: {
+                    query: {
+                      term: {
+                        text: {
+                          value: "term",
+                          _name: "my_query"
+                        }
+                      }
+                    }
+                  }
+                },
+                {
+                  knn: {
+                    field: "vector",
+                    query_vector: [ 3 ],
+                    num_candidates: 5,
+                    k: 5,
+                    _name: "my_top_knn"
+                  }
+                }
+              ]
+              rank_window_size: 5
+              rank_constant: 1
+          size: 3
+          explain: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - close_to: { hits.hits.0._explanation.value: { value: 0.8333334, error: 0.000001 } }
+  - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.8333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.value: 2}
+  - match: {hits.hits.0._explanation.details.0.description: "/.*my_query.*/" }
+  - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.0._explanation.details.1.value: 1}
+  - match: {hits.hits.0._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } }
+  - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.value: 3}
+  - match: {hits.hits.1._explanation.details.0.description: "/.*my_query.*/" }
+  - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.1._explanation.details.1.value: 2}
+  - match: {hits.hits.1._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - match: {hits.hits.2._explanation.value: 0.5}
+  - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.value: 1}
+  - match: {hits.hits.2._explanation.details.0.description: "/.*my_query.*/" }
+  - match: {hits.hits.2._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.2._explanation.details.1.value: 0}
+  - match: {hits.hits.2._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - length: {hits.hits.2._explanation.details.1.details: 0}
+
+---
+"using a mix of named and unnamed retrievers":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          retriever:
+            rrf:
+              retrievers: [
+                {
+                  standard: {
+                    query: {
+                      term: {
+                        text: {
+                          value: "term"
+                        }
+                      }
+                    }
+                  }
+                },
+                {
+                  knn: {
+                    field: "vector",
+                    query_vector: [ 3 ],
+                    num_candidates: 5,
+                    k: 5,
+                    _name: "my_top_knn"
+                  }
+                }
+              ]
+              rank_window_size: 5
+              rank_constant: 1
+          size: 3
+          explain: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - close_to: { hits.hits.0._explanation.value: { value: 0.8333334, error: 0.000001 } }
+  - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.8333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.value: 2}
+  - match: {hits.hits.0._explanation.details.0.description: "/.*at.index.\\[0\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.0._explanation.details.1.value: 1}
+  - match: {hits.hits.0._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } }
+  - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.value: 3}
+  - match: {hits.hits.1._explanation.details.0.description: "/.*at.index.\\[0\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.1._explanation.details.1.value: 2}
+  - match: {hits.hits.1._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - match: {hits.hits.2._explanation.value: 0.5}
+  - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.value: 1}
+  - match: {hits.hits.2._explanation.details.0.description: "/.*at.index.\\[0\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.2._explanation.details.1.value: 0}
+  - match: {hits.hits.2._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - length: {hits.hits.2._explanation.details.1.details: 0}
diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml
new file mode 100644
index 0000000000000..5718cd3455526
--- /dev/null
+++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml
@@ -0,0 +1,386 @@
+setup:
+  - requires:
+      cluster_features: "gte_v8.15.0"
+      reason: 'explain for rrf was added in 8.15'
+      test_runner_features: close_to
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              text:
+                type: text
+              integer:
+                type: integer
+              vector:
+                type: dense_vector
+                dims: 1
+                index: true
+                similarity: l2_norm
+                index_options:
+                  type: hnsw
+                  ef_construction: 100
+                  m: 16
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          text: "term"
+          integer: 1
+          vector: [5]
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          text: "term term"
+          integer: 2
+          vector: [4]
+
+  - do:
+      index:
+        index: test
+        id: "3"
+        body:
+          text: "term term term"
+          integer: 3
+          vector: [3]
+  - do:
+      index:
+        index: test
+        id: "4"
+        body:
+          text: "term term term term"
+          integer: 3
+
+  - do:
+      index:
+        index: test
+        id: "5"
+        body:
+          integer: 1
+          vector: [0]
+
+  - do:
+      indices.refresh: {}
+
+---
+"using a top level knn and query":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          knn:
+            field: vector
+            query_vector: [3]
+            k: 5
+            num_candidates: 5
+          query:
+            term:
+              text: term
+          rank:
+            rrf:
+              rank_window_size: 5
+              rank_constant: 1
+          size: 3
+          explain: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - close_to: { hits.hits.0._explanation.value: { value: 0.8333334, error: 0.000001 } }
+  - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.8333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.value: 2}
+  - match: {hits.hits.0._explanation.details.0.description: "/rrf.score:.\\[0.33333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.0._explanation.details.1.value: 1}
+  - match: {hits.hits.0._explanation.details.1.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } }
+  - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.value: 3}
+  - match: {hits.hits.1._explanation.details.0.description: "/rrf.score:.\\[0.25\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.1._explanation.details.1.value: 2}
+  - match: {hits.hits.1._explanation.details.1.description: "/rrf.score:.\\[0.33333334\\].*/" }
+  - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - match: {hits.hits.2._explanation.value: 0.5}
+  - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.value: 1}
+  - match: {hits.hits.2._explanation.details.0.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.2._explanation.details.1.value: 0}
+  - match: {hits.hits.2._explanation.details.1.description: "/rrf.score:.\\[0\\],.result.not.found./"}
+  - length: {hits.hits.2._explanation.details.1.details: 0}
+
+---
+"using sub_searches":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          sub_searches: [
+            {
+              "query": {
+                "term": {
+                  "text": "term"
+                }
+              }
+            },
+            {
+              "query": {
+                "knn":
+                  {
+                    "field": "vector",
+                    "query_vector": [ 3 ],
+                    "num_candidates": 5
+                  }
+              }
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 5
+              rank_constant: 1
+          size: 3
+          explain: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - close_to: { hits.hits.0._explanation.value: { value: 0.8333334, error: 0.000001 } }
+  - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.8333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.value: 2}
+  - match: {hits.hits.0._explanation.details.0.description: "/rrf.score:.\\[0.33333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.0._explanation.details.1.value: 1}
+  - match: {hits.hits.0._explanation.details.1.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } }
+  - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.value: 3}
+  - match: {hits.hits.1._explanation.details.0.description: "/rrf.score:.\\[0.25\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.1._explanation.details.1.value: 2}
+  - match: {hits.hits.1._explanation.details.1.description: "/rrf.score:.\\[0.33333334\\].*/" }
+  - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - match: {hits.hits.2._explanation.value: 0.5}
+  - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.value: 1}
+  - match: {hits.hits.2._explanation.details.0.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.2._explanation.details.1.value: 0}
+  - match: {hits.hits.2._explanation.details.1.description: "/rrf.score:.\\[0\\],.result.not.found./"}
+  - length: {hits.hits.2._explanation.details.1.details: 0}
+
+---
+"using named top level knn and query":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          knn:
+            field: vector
+            query_vector: [3]
+            k: 5
+            num_candidates: 5
+            _name: my_top_knn
+          query:
+            term:
+              text:
+                value: term
+                _name: my_query
+          rank:
+            rrf:
+              rank_window_size: 5
+              rank_constant: 1
+          size: 3
+          explain: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - close_to: { hits.hits.0._explanation.value: { value: 0.8333334, error: 0.000001 } }
+  - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.8333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.value: 2}
+  - match: {hits.hits.0._explanation.details.0.description: "/.*my_query.*/" }
+  - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.0._explanation.details.1.value: 1}
+  - match: {hits.hits.0._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } }
+  - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.value: 3}
+  - match: {hits.hits.1._explanation.details.0.description: "/.*my_query.*/" }
+  - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.1._explanation.details.1.value: 2}
+  - match: {hits.hits.1._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - match: {hits.hits.2._explanation.value: 0.5}
+  - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.value: 1}
+  - match: {hits.hits.2._explanation.details.0.description: "/.*my_query.*/" }
+  - match: {hits.hits.2._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.2._explanation.details.1.value: 0}
+  - match: {hits.hits.2._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - length: {hits.hits.2._explanation.details.1.details: 0}
+
+---
+"using named sub_searches":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          sub_searches: [
+            {
+              "query": {
+                "term": {
+                  "text": {
+                    "value": "term",
+                    "_name": "my_query"
+                  }
+                }
+              }
+            },
+            {
+              "query": {
+                "knn":
+                  {
+                    "field": "vector",
+                    "query_vector": [ 3 ],
+                    "num_candidates": 5,
+                    "_name": "my_top_knn"
+                  }
+              }
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 5
+              rank_constant: 1
+          size: 3
+          explain: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - close_to: { hits.hits.0._explanation.value: { value: 0.8333334, error: 0.000001 } }
+  - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.8333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.value: 2}
+  - match: {hits.hits.0._explanation.details.0.description: "/.*my_query.*/" }
+  - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.0._explanation.details.1.value: 1}
+  - match: {hits.hits.0._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } }
+  - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.value: 3}
+  - match: {hits.hits.1._explanation.details.0.description: "/.*my_query.*/" }
+  - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.1._explanation.details.1.value: 2}
+  - match: {hits.hits.1._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - match: {hits.hits.2._explanation.value: 0.5}
+  - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.value: 1}
+  - match: {hits.hits.2._explanation.details.0.description: "/.*my_query.*/" }
+  - match: {hits.hits.2._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.2._explanation.details.1.value: 0}
+  - match: {hits.hits.2._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - length: {hits.hits.2._explanation.details.1.details: 0}
+
+---
+"using a mix of named and unnamed queries":
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "text", "integer" ]
+          sub_searches: [
+            {
+              "query": {
+                "term": {
+                  "text": {
+                    "value": "term"
+                  }
+                }
+              }
+            },
+            {
+              "query": {
+                "knn":
+                  {
+                    "field": "vector",
+                    "query_vector": [ 3 ],
+                    "num_candidates": 5,
+                    "_name": "my_top_knn"
+                  }
+              }
+            }
+          ]
+          rank:
+            rrf:
+              rank_window_size: 5
+              rank_constant: 1
+          size: 3
+          explain: true
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.2._id: "4" }
+
+  - close_to: { hits.hits.0._explanation.value: { value: 0.8333334, error: 0.000001 } }
+  - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.8333334\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.value: 2}
+  - match: {hits.hits.0._explanation.details.0.description: "/.*at.index.\\[0\\].*/" }
+  - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.0._explanation.details.1.value: 1}
+  - match: {hits.hits.0._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } }
+  - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.value: 3}
+  - match: {hits.hits.1._explanation.details.0.description: "/.*at.index.\\[0\\].*/" }
+  - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.1._explanation.details.1.value: 2}
+  - match: {hits.hits.1._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" }
+
+  - match: {hits.hits.2._explanation.value: 0.5}
+  - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.value: 1}
+  - match: {hits.hits.2._explanation.details.0.description: "/.*at.index.\\[0\\].*/" }
+  - match: {hits.hits.2._explanation.details.0.details.0.description: "/weight\\(text:term.*/" }
+  - match: {hits.hits.2._explanation.details.1.value: 0}
+  - match: {hits.hits.2._explanation.details.1.description: "/.*my_top_knn.*/" }
+  - length: {hits.hits.2._explanation.details.1.details: 0}

From 40d88a270b142d083c9344effb746eda8861fa5c Mon Sep 17 00:00:00 2001
From: Henning Andersen <33268011+henningandersen@users.noreply.github.com>
Date: Fri, 7 Jun 2024 11:47:47 +0200
Subject: [PATCH 208/208] Reduce stateless default refresh to 5s (#109393)

Having done recent optimizations, we can now start reducing the
refresh interval for stateless. As a first step we will reduce
the default to 5s.

Relates ES-7774
---
 server/src/main/java/org/elasticsearch/index/IndexSettings.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
index aa92025f32428..5446027a2ca40 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -284,7 +284,7 @@ public final class IndexSettings {
         TimeValue.MINUS_ONE,
         Property.NodeScope
     ); // TODO: remove setting
-    public static TimeValue STATELESS_DEFAULT_REFRESH_INTERVAL = TimeValue.timeValueSeconds(15); // TODO: this value is still not final
+    public static TimeValue STATELESS_DEFAULT_REFRESH_INTERVAL = TimeValue.timeValueSeconds(5); // TODO: this value is still not final
     public static TimeValue STATELESS_MIN_NON_FAST_REFRESH_INTERVAL = TimeValue.timeValueSeconds(5);
     public static final Setting<TimeValue> INDEX_REFRESH_INTERVAL_SETTING = Setting.timeSetting("index.refresh_interval", (settings) -> {
         if (EXISTING_SHARDS_ALLOCATOR_SETTING.get(settings).equals("stateless") && INDEX_FAST_REFRESH_SETTING.get(settings) == false) {