Merge branch 'main' into inference-chunks-offsets

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.template.yml

@@ -10,6 +10,7 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204

.buildkite/pipelines/periodic-packaging.yml

@@ -11,6 +11,7 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
@@ -301,8 +302,8 @@ steps:
         env:
           BWC_VERSION: 8.16.2
 
-      - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0
+      - label: "{{matrix.image}} / 8.17.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -315,7 +316,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.0
+          BWC_VERSION: 8.17.1
 
       - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0

.buildkite/pipelines/periodic-platform-support.yml

@@ -10,6 +10,7 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204

.buildkite/pipelines/periodic.yml

@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.0#bwcTest
+      - label: 8.17.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.0
+          BWC_VERSION: 8.17.1
         retry:
           automatic:
             - exit_status: "-1"
@@ -448,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -490,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -13,6 +13,7 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204

.buildkite/pipelines/pull-request/part-1-entitlements.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-1-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart1
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-1.yml

@@ -1,6 +1,8 @@
 steps:
   - label: part-1
-    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1
+    command: |
+      .buildkite/scripts/spotless.sh # This doesn't have to be part of part-1, it was just a convenient place to put it
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1
     timeout_in_minutes: 300
     agents:
       provider: gcp

.buildkite/pipelines/pull-request/part-2-entitlements.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-2-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart2
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-3-entitlements.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-3-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart3
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-4-entitlements.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-4-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-5-entitlements.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-5-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart5
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/precommit.yml

@@ -3,7 +3,9 @@ config:
   skip-labels: []
 steps:
   - label: precommit
-    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit
+    command: |
+      .buildkite/scripts/spotless.sh
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit
     timeout_in_minutes: 300
     agents:
       provider: gcp

.buildkite/scripts/gradle-configuration-cache-validation.sh

@@ -3,16 +3,16 @@
 set -euo pipefail
 
 # This is a workaround for https://github.com/gradle/gradle/issues/28159
-.ci/scripts/run-gradle.sh --no-daemon precommit
+.ci/scripts/run-gradle.sh --no-daemon $@
 
-.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
+.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@
 
 # Create a temporary file
 tmpOutputFile=$(mktemp)
 trap "rm $tmpOutputFile" EXIT
 
 echo "2nd run"
-.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
+.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@ | tee $tmpOutputFile
 
 # Check if the command was successful
 if grep -q "Configuration cache entry reused." $tmpOutputFile; then

.buildkite/scripts/spotless.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+if [[ -z "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+  echo "Not a pull request, skipping spotless"
+  exit 0
+fi
+
+if ! git diff --exit-code; then
+  echo "Changes are present before running spotless, not running"
+  git status
+  exit 0
+fi
+
+NEW_COMMIT_MESSAGE="[CI] Auto commit changes from spotless"
+PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
+
+echo "--- Running spotless"
+.ci/scripts/run-gradle.sh -Dscan.tag.NESTED spotlessApply
+
+if git diff --exit-code; then
+  echo "No changes found after running spotless. Don't need to auto commit."
+  exit 0
+fi
+
+if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then
+  echo "Changes found after running spotless"
+  echo "CI already attempted to commit these changes, but the file(s) seem to have changed again."
+  echo "Please review and fix manually."
+  exit 1
+fi
+
+git config --global user.name elasticsearchmachine
+git config --global user.email 'infra-root+elasticsearchmachine@elastic.co'
+
+gh pr checkout "${BUILDKITE_PULL_REQUEST}"
+git add -u .
+git commit -m "$NEW_COMMIT_MESSAGE"
+git push
+
+# After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled
+# So, let's just sleep to give the build time to cancel itself without an error
+# If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either)
+sleep 300
+exit 1

.ci/bwcVersions

@@ -16,6 +16,6 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.5"
   - "8.16.2"
-  - "8.17.0"
+  - "8.17.1"
   - "8.18.0"
   - "9.0.0"

.ci/scripts/run-gradle.sh

@@ -5,35 +5,39 @@ mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
 
 MAX_WORKERS=4
 
-# Don't run this stuff on Windows
-if ! uname -a | grep -q MING; then
-  # drop page cache and kernel slab objects on linux
-  [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
+if [ -z "$MAX_WORKER_ENV" ]; then
+    # Don't run this stuff on Windows
+    if ! uname -a | grep -q MING; then
+      # drop page cache and kernel slab objects on linux
+      [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
 
-  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
-    MAX_WORKERS=16
-  elif [ -f /proc/cpuinfo ]; then
-    MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
-  else
-    if [[ "$OSTYPE" == "darwin"* ]]; then
-        MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
-    else
-        echo "Unsupported OS Type: $OSTYPE"
-        exit 1
-    fi
-  fi
-  if pwd | grep -v -q ^/dev/shm ; then
-    echo "Not running on a ramdisk, reducing number of workers"
-    MAX_WORKERS=$(($MAX_WORKERS*2/3))
-  fi
+      if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+        MAX_WORKERS=16
+      elif [ -f /proc/cpuinfo ]; then
+        MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
+      else
+        if [[ "$OSTYPE" == "darwin"* ]]; then
+            MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
+        else
+            echo "Unsupported OS Type: $OSTYPE"
+            exit 1
+        fi
+      fi
+      if pwd | grep -v -q ^/dev/shm ; then
+        echo "Not running on a ramdisk, reducing number of workers"
+        MAX_WORKERS=$(($MAX_WORKERS*2/3))
+      fi
 
-  # Export glibc version as environment variable since some BWC tests are incompatible with later versions
-  export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
-fi
+      # Export glibc version as environment variable since some BWC tests are incompatible with later versions
+      export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
+    fi
 
-# Running on 2-core machines without ramdisk can make this value be 0
-if [[ "$MAX_WORKERS" == "0" ]]; then
-  MAX_WORKERS=1
+    # Running on 2-core machines without ramdisk can make this value be 0
+    if [[ "$MAX_WORKERS" == "0" ]]; then
+      MAX_WORKERS=1
+    fi
+else
+    MAX_WORKERS=$MAX_WORKER_ENV
 fi
 
 set -e

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "8.16.2"
-  - "8.17.0"
+  - "8.17.1"
   - "8.18.0"
   - "9.0.0"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -16,6 +16,7 @@
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
+import org.gradle.api.Task;
 import org.gradle.api.logging.LogLevel;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
@@ -103,10 +104,16 @@ private static TaskProvider<LoggedExec> createRunBwcGradleTask(
             loggedExec.dependsOn("checkoutBwcBranch");
             loggedExec.getWorkingDir().set(checkoutDir.get());
 
-            loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", providerFactory.of(JavaHomeValueSource.class, spec -> {
-                spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version()));
-                spec.getParameters().getCheckoutDir().set(checkoutDir);
-            }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s))));
+            loggedExec.doFirst(new Action<Task>() {
+                @Override
+                public void execute(Task task) {
+                    Provider<String> minimumCompilerVersionValueSource = providerFactory.of(JavaHomeValueSource.class, spec -> {
+                        spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version()));
+                        spec.getParameters().getCheckoutDir().set(checkoutDir);
+                    }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s)));
+                    loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", minimumCompilerVersionValueSource.get());
+                }
+            });
 
             if (isCi && OS.current() != OS.WINDOWS) {
                 // TODO: Disabled for now until we can figure out why files are getting corrupted
@@ -169,14 +176,6 @@ private static Provider<String> getJavaHome(ObjectFactory objectFactory, JavaToo
             .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath());
     }
 
-    private static String readFromFile(File file) {
-        try {
-            return FileUtils.readFileToString(file).trim();
-        } catch (IOException ioException) {
-            throw new GradleException("Cannot read java properties file.", ioException);
-        }
-    }
-
     public abstract static class JavaHomeValueSource implements ValueSource<String, JavaHomeValueSource.Params> {
 
         private String minimumCompilerVersionPath(Version bwcVersion) {
@@ -185,6 +184,14 @@ private String minimumCompilerVersionPath(Version bwcVersion) {
                 : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
         }
 
+        private static String readFromFile(File file) {
+            try {
+                return FileUtils.readFileToString(file).trim();
+            } catch (IOException ioException) {
+                throw new GradleException("Cannot read java properties file.", ioException);
+            }
+        }
+
         @Override
         public String obtain() {
             return readFromFile(

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:32f06b169bb4b0f257fbb10e8c8379f06d3ee1355c89b3327cb623781a29590e",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:1b51ff6dba78c98d3e02b0cd64a8ce3238c7a40408d21e3af12a329d44db6f23",
         "-wolfi",
         "apk"
     ),