Merge remote-tracking branch 'upstream/main' into ps230914-bumpTV-add…

…RecoveryCommitTooNewException

.buildkite/hooks/pre-command

@@ -74,3 +74,8 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
   SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk)
   export SNYK_TOKEN
 fi
+
+if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then
+  # Run in the background, while the job continues
+  nohup .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &
+fi

.buildkite/pipelines/intake.yml

@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.14", "8.10.1", "8.11.0"]
+            BWC_VERSION: ["7.17.14", "8.10.2", "8.11.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1616,6 +1616,22 @@ steps:
         env:
           BWC_VERSION: 8.10.1
 
+      - label: "{{matrix.image}} / 8.10.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.2
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.2
+
       - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -992,6 +992,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.1
+      - label: 8.10.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.2#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.2
       - label: 8.11.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300

.buildkite/scripts/setup-monitoring.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ELASTIC_AGENT_URL=$(vault read -field=url secret/ci/elastic-elasticsearch/elastic-agent-token)
+ELASTIC_AGENT_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/elastic-agent-token)
+
+if [[ ! -d /opt/elastic-agent ]]; then
+  sudo mkdir /opt/elastic-agent
+  sudo chown -R buildkite-agent:buildkite-agent /opt/elastic-agent
+  cd /opt/elastic-agent
+
+  archive=elastic-agent-8.10.1-linux-x86_64.tar.gz
+  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+    archive=elastic-agent-8.10.1-linux-arm64.tar.gz
+  fi
+
+  curl -L -O "https://artifacts.elastic.co/downloads/beats/elastic-agent/$archive"
+
+  tar xzf "$archive" --directory=. --strip-components=1
+fi
+
+cd /opt/elastic-agent
+sudo ./elastic-agent install -f --url="$ELASTIC_AGENT_URL" --enrollment-token="$ELASTIC_AGENT_TOKEN"

.ci/bwcVersions

@@ -98,4 +98,5 @@ BWC_VERSION:
   - "8.9.2"
   - "8.10.0"
   - "8.10.1"
+  - "8.10.2"
   - "8.11.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.14"
-  - "8.10.1"
+  - "8.10.2"
   - "8.11.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java

@@ -9,6 +9,9 @@
 package org.elasticsearch.benchmark.compute.operator;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -20,6 +23,8 @@
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.compute.operator.topn.TopNEncoder;
 import org.elasticsearch.compute.operator.topn.TopNOperator;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -96,7 +101,13 @@ private static Operator operator(String data, int topCount) {
             case LONGS_AND_BYTES_REFS -> List.of(TopNEncoder.DEFAULT_SORTABLE, TopNEncoder.UTF8);
             default -> throw new IllegalArgumentException("unsupported data type [" + data + "]");
         };
+        CircuitBreakerService breakerService = new HierarchyCircuitBreakerService(
+            Settings.EMPTY,
+            List.of(),
+            ClusterSettings.createBuiltInClusterSettings()
+        );
         return new TopNOperator(
+            breakerService.getBreaker(CircuitBreaker.REQUEST),
             topCount,
             elementTypes,
             encoders,

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -152,7 +152,7 @@ public void setUp() throws Exception {
             .metadata(metadata)
             .routingTable(routingTable)
             .nodes(nb)
-            .compatibilityVersions(compatibilityVersions)
+            .nodeIdsToCompatibilityVersions(compatibilityVersions)
             .build();
     }
 

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy

@@ -172,6 +172,7 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl
 
     def "applies conventions on yaml-rest-test tests"() {
         given:
+        buildApiRestrictionsDisabled = true
         clazz(dir('src/yamlRestTest/java'), "org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase")
         buildFile << """
         apply plugin:'elasticsearch.legacy-yaml-rest-test'
@@ -211,6 +212,7 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl
     @Unroll
     def "applies conventions on #sourceSetName tests"() {
         given:
+        buildApiRestrictionsDisabled = pluginName.contains('legacy')
         clazz(dir("src/${sourceSetName}/java"), "org.elasticsearch.test.ESIntegTestCase")
         clazz(dir("src/${sourceSetName}/java"), "org.elasticsearch.test.rest.ESRestTestCase")
         buildFile << """

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy

@@ -33,7 +33,9 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         // 1. TestClustersPlugin not cc compatible due to listener registration
         // 2. RestIntegTestTask not cc compatible due to
         configurationCacheCompatible = false
+        buildApiRestrictionsDisabled = true
     }
+
     def "yamlRestTestVxCompatTest does nothing when there are no tests"() {
         given:
         subProject(":distribution:bwc:maintenance") << """

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy

@@ -17,6 +17,11 @@ import org.gradle.testkit.runner.TaskOutcome
 @IgnoreIf({ os.isWindows() })
 class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
+    def setup() {
+        buildApiRestrictionsDisabled = true
+    }
+
+
     def "yamlRestTest does nothing when there are no tests"() {
         given:
         // RestIntegTestTask not cc compatible due to