Merge remote-tracking branch 'upstream/main' into ps240828-makeWriteP…

…oolEwmaConfigurable

benchmarks/build.gradle

@@ -1,4 +1,5 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.TestUtil
 
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
@@ -29,6 +30,7 @@ tasks.named("javadoc").configure { enabled = false }
 configurations {
   expression
   painless
+  nativeLib
 }
 
 dependencies {
@@ -45,6 +47,7 @@ dependencies {
   implementation project(path: ':libs:elasticsearch-simdvec')
   expression(project(path: ':modules:lang-expression', configuration: 'zip'))
   painless(project(path: ':modules:lang-painless', configuration: 'zip'))
+  nativeLib(project(':libs:elasticsearch-native'))
   api "org.openjdk.jmh:jmh-core:$versions.jmh"
   annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
   // Dependencies of JMH
@@ -76,17 +79,8 @@ tasks.register("copyPainless", Copy) {
 tasks.named("run").configure {
   executable = "${BuildParams.runtimeJavaHome}/bin/java"
   args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index"
-  dependsOn "copyExpression", "copyPainless"
-  systemProperty 'es.nativelibs.path', file("../libs/native/libraries/build/platform/${platformName()}-${os.arch}")
-}
-
-String platformName() {
-  String name = System.getProperty("os.name");
-  if (name.startsWith("Mac")) {
-    return "darwin";
-  } else {
-    return name.toLowerCase(Locale.ROOT);
-  }
+  dependsOn "copyExpression", "copyPainless", configurations.nativeLib
+  systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())
 }
 
 spotless {

distribution/docker/src/docker/Dockerfile

@@ -22,7 +22,7 @@
 <% if (docker_base == 'iron_bank') { %>
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9
-ARG BASE_TAG=9.3
+ARG BASE_TAG=9.4
 <% } %>
 
 ################################################################################

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -14,7 +14,7 @@ tags:
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.3"
+  BASE_TAG: "9.4"
 
 # Docker image labels
 labels:

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java

@@ -32,6 +32,7 @@
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Locale;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * The main CLI for running Elasticsearch.
@@ -44,6 +45,8 @@ class ServerCli extends EnvironmentAwareCommand {
     private final OptionSpecBuilder quietOption;
     private final OptionSpec<String> enrollmentTokenOption;
 
+    // flag for indicating shutdown has begun. we use an AtomicBoolean to double as a synchronization object
+    private final AtomicBoolean shuttingDown = new AtomicBoolean(false);
     private volatile ServerProcess server;
 
     // visible for testing
@@ -98,7 +101,14 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
             syncPlugins(terminal, env, processInfo);
 
             ServerArgs args = createArgs(options, env, secrets, processInfo);
-            this.server = startServer(terminal, processInfo, args);
+            synchronized (shuttingDown) {
+                // if we are shutting down there is no reason to start the server
+                if (shuttingDown.get()) {
+                    terminal.println("CLI is shutting down, skipping starting server process");
+                    return;
+                }
+                this.server = startServer(terminal, processInfo, args);
+            }
         }
 
         if (options.has(daemonizeOption)) {
@@ -233,8 +243,11 @@ private ServerArgs createArgs(OptionSet options, Environment env, SecureSettings
 
     @Override
     public void close() throws IOException {
-        if (server != null) {
-            server.stop();
+        synchronized (shuttingDown) {
+            shuttingDown.set(true);
+            if (server != null) {
+                server.stop();
+            }
         }
     }
 

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java

@@ -36,6 +36,8 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
@@ -50,6 +52,7 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.matchesRegex;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class ServerCliTests extends CommandTestCase {
 
@@ -383,6 +386,52 @@ public void testSecureSettingsLoaderWithNullPassword() throws Exception {
         assertEquals("", loader.password);
     }
 
+    public void testProcessCreationRace() throws Exception {
+        for (int i = 0; i < 10; ++i) {
+            CyclicBarrier raceStart = new CyclicBarrier(2);
+            TestServerCli cli = new TestServerCli() {
+                @Override
+                void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception {
+                    super.syncPlugins(terminal, env, processInfo);
+                    raceStart.await();
+                }
+
+                @Override
+                public void close() throws IOException {
+                    try {
+                        raceStart.await();
+                    } catch (InterruptedException ie) {
+                        Thread.currentThread().interrupt();
+                        throw new AssertionError(ie);
+                    } catch (BrokenBarrierException e) {
+                        throw new AssertionError(e);
+                    }
+                    super.close();
+                }
+            };
+            Thread closeThread = new Thread(() -> {
+                try {
+                    cli.close();
+                } catch (IOException e) {
+                    throw new AssertionError(e);
+                }
+            });
+            closeThread.start();
+            cli.main(new String[] {}, terminal, new ProcessInfo(sysprops, envVars, esHomeDir));
+            closeThread.join();
+
+            if (cli.getServer() == null) {
+                // close won the race, so server should never have been started
+                assertThat(cli.startServerCalled, is(false));
+            } else {
+                // creation won the race, so check we correctly waited on it and stopped
+                assertThat(cli.getServer(), sameInstance(mockServer));
+                assertThat(mockServer.waitForCalled, is(true));
+                assertThat(mockServer.stopCalled, is(true));
+            }
+        }
+    }
+
     private MockSecureSettingsLoader loadWithMockSecureSettingsLoader() throws Exception {
         var loader = new MockSecureSettingsLoader();
         this.mockSecureSettingsLoader = loader;
@@ -465,9 +514,9 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
     }
 
     private class MockServerProcess extends ServerProcess {
-        boolean detachCalled = false;
-        boolean waitForCalled = false;
-        boolean stopCalled = false;
+        volatile boolean detachCalled = false;
+        volatile boolean waitForCalled = false;
+        volatile boolean stopCalled = false;
 
         MockServerProcess() {
             super(null, null);
@@ -505,6 +554,8 @@ void reset() {
     }
 
     private class TestServerCli extends ServerCli {
+        boolean startServerCalled = false;
+
         @Override
         protected Command loadTool(String toolname, String libs) {
             if (toolname.equals("auto-configure-node")) {
@@ -551,20 +602,21 @@ protected SecureSettingsLoader secureSettingsLoader(Environment env) {
 
             return new KeystoreSecureSettingsLoader();
         }
+
+        @Override
+        protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception {
+            startServerCalled = true;
+            if (argsValidator != null) {
+                argsValidator.accept(args);
+            }
+            mockServer.reset();
+            return mockServer;
+        }
     }
 
     @Override
     protected Command newCommand() {
-        return new TestServerCli() {
-            @Override
-            protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) {
-                if (argsValidator != null) {
-                    argsValidator.accept(args);
-                }
-                mockServer.reset();
-                return mockServer;
-            }
-        };
+        return new TestServerCli();
     }
 
     static class MockSecureSettingsLoader implements SecureSettingsLoader {

docs/changelog/111818.yaml

@@ -0,0 +1,5 @@
+pr: 111818
+summary: Add tier preference to security index settings allowlist
+area: Security
+type: enhancement
+issues: []

docs/changelog/111932.yaml

@@ -0,0 +1,6 @@
+pr: 111932
+summary: Fix union-types where one index is missing the field
+area: ES|QL
+type: bug
+issues:
+ - 111912

docs/changelog/112066.yaml

@@ -0,0 +1,6 @@
+pr: 112066
+summary: Do not treat replica as unassigned if primary recently created and unassigned
+  time is below a threshold
+area: Health
+type: enhancement
+issues: []

docs/changelog/112151.yaml

@@ -0,0 +1,5 @@
+pr: 112151
+summary: Store original source for keywords using a normalizer
+area: Logs
+type: enhancement
+issues: []

docs/changelog/112178.yaml

@@ -0,0 +1,6 @@
+pr: 112178
+summary: Avoid wrapping rejection exception in exchange
+area: ES|QL
+type: bug
+issues:
+ - 112106

docs/changelog/112242.yaml

@@ -0,0 +1,5 @@
+pr: 112242
+summary: Fix toReleaseVersion() when called on the current version id
+area: Infra/Core
+type: bug
+issues: [111900]

docs/changelog/112260.yaml

@@ -0,0 +1,6 @@
+pr: 112260
+summary: Fix DLS over Runtime Fields
+area: "Authorization"
+type: bug
+issues:
+ - 111637

docs/changelog/112270.yaml

@@ -0,0 +1,5 @@
+pr: 112270
+summary: Support sparse embedding models in the elasticsearch inference service
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/112273.yaml

@@ -0,0 +1,5 @@
+pr: 111181
+summary: "[Inference API] Add Docs for AlibabaCloud AI Search Support for the Inference API"
+area: Machine Learning
+type: enhancement
+issues: [ ]

docs/changelog/112277.yaml

@@ -0,0 +1,5 @@
+pr: 112277
+summary: Upgrade `repository-azure` dependencies
+area: Snapshot/Restore
+type: upgrade
+issues: []

docs/changelog/112320.yaml

@@ -0,0 +1,5 @@
+pr: 112320
+summary: Upgrade xcontent to Jackson 2.17.2
+area: Infra/Core
+type: upgrade
+issues: []

docs/changelog/112341.yaml

@@ -0,0 +1,5 @@
+pr: 112341
+summary: Fix DLS using runtime fields and synthetic source
+area: Authorization
+type: bug
+issues: []

docs/reference/inference/inference-apis.asciidoc

@@ -39,6 +39,7 @@ include::delete-inference.asciidoc[]
 include::get-inference.asciidoc[]
 include::post-inference.asciidoc[]
 include::put-inference.asciidoc[]
+include::service-alibabacloud-ai-search.asciidoc[]
 include::service-amazon-bedrock.asciidoc[]
 include::service-anthropic.asciidoc[]
 include::service-azure-ai-studio.asciidoc[]

docs/reference/inference/put-inference.asciidoc

@@ -39,6 +39,7 @@ The create {infer} API enables you to create an {infer} endpoint and configure a
 
 The following services are available through the {infer} API, click the links to review the configuration details of the services:
 
+* <<infer-service-alibabacloud-ai-search,AlibabaCloud AI Search>>
 * <<infer-service-amazon-bedrock,Amazon Bedrock>>
 * <<infer-service-anthropic,Anthropic>>
 * <<infer-service-azure-ai-studio,Azure AI Studio>>