Merge branch 'main' into update-service-in-tutorial

# Conflicts:
#	docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc
#	docs/reference/search/search-your-data/semantic-text-hybrid-search

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic.yml

@@ -448,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -490,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/snapshotBwcVersions

@@ -1,4 +1,5 @@
 BWC_VERSION:
+  - "8.15.6"
   - "8.16.2"
   - "8.17.0"
   - "8.18.0"

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy

@@ -9,9 +9,10 @@
 
 package org.elasticsearch.gradle.internal
 
+import spock.lang.Unroll
+
 import org.elasticsearch.gradle.fixtures.AbstractGitAwareGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
-import spock.lang.Unroll
 
 class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest {
 
@@ -23,8 +24,10 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
             apply plugin: 'elasticsearch.internal-distribution-bwc-setup'
         """
         execute("git branch origin/8.x", file("cloned"))
+        execute("git branch origin/8.3", file("cloned"))
+        execute("git branch origin/8.2", file("cloned"))
+        execute("git branch origin/8.1", file("cloned"))
         execute("git branch origin/7.16", file("cloned"))
-        execute("git branch origin/7.15", file("cloned"))
     }
 
     def "builds distribution from branches via archives extractedAssemble"() {
@@ -48,10 +51,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
         assertOutputContains(result.output, "[$bwcDistVersion] > Task :distribution:archives:darwin-tar:${expectedAssembleTaskName}")
 
         where:
-        bwcDistVersion | bwcProject | expectedAssembleTaskName
-        "8.0.0"        | "minor"    | "extractedAssemble"
-        "7.16.0"       | "staged"   | "extractedAssemble"
-        "7.15.2"       | "bugfix"   | "extractedAssemble"
+        bwcDistVersion | bwcProject    | expectedAssembleTaskName
+        "8.4.0"        | "minor"       | "extractedAssemble"
+        "8.3.0"        | "staged"      | "extractedAssemble"
+        "8.2.1"        | "bugfix"      | "extractedAssemble"
+        "8.1.3"        | "bugfix2"     | "extractedAssemble"
     }
 
     @Unroll
@@ -70,8 +74,8 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
 
         where:
         bwcDistVersion | platform
-        "8.0.0"       | "darwin"
-        "8.0.0"       | "linux"
+        "8.4.0"       | "darwin"
+        "8.4.0"       | "linux"
     }
 
     def "bwc expanded distribution folder can be resolved as bwc project artifact"() {
@@ -107,11 +111,11 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
         result.task(":resolveExpandedDistribution").outcome == TaskOutcome.SUCCESS
         result.task(":distribution:bwc:minor:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
         and: "assemble task triggered"
-        result.output.contains("[8.0.0] > Task :distribution:archives:darwin-tar:extractedAssemble")
+        result.output.contains("[8.4.0] > Task :distribution:archives:darwin-tar:extractedAssemble")
         result.output.contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-8.x/" +
                         "distribution/archives/darwin-tar/build/install")
         result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.x/" +
-                        "distribution/archives/darwin-tar/build/install/elasticsearch-8.0.0-SNAPSHOT")
+                        "distribution/archives/darwin-tar/build/install/elasticsearch-8.4.0-SNAPSHOT")
     }
 
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -57,7 +57,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
 
             elasticsearch_distributions {
               test_distro {
-                  version = "8.0.0"
+                  version = "8.4.0"
                   type = "archive"
                   platform = "linux"
                   architecture = Architecture.current();
@@ -87,7 +87,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
 
             elasticsearch_distributions {
               test_distro {
-                  version = "8.0.0"
+                  version = "8.4.0"
                   type = "archive"
                   platform = "linux"
                   architecture = Architecture.current();

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy

@@ -40,7 +40,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         given:
         internalBuild()
 
-        subProject(":distribution:bwc:staged") << """
+        subProject(":distribution:bwc:minor") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -61,11 +61,11 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
     }
 
-    def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:staged"() {
+    def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:minor"() {
         given:
         internalBuild()
 
-        subProject(":distribution:bwc:staged") << """
+        subProject(":distribution:bwc:minor") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -98,8 +98,8 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         String api = "foo.json"
         String test = "10_basic.yml"
         //add the compatible test and api files, these are the prior version's normal yaml rest tests
-        file("distribution/bwc/staged/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
-        file("distribution/bwc/staged/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
+        file("distribution/bwc/minor/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
+        file("distribution/bwc/minor/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
 
         when:
         def result = gradleRunner("yamlRestCompatTest").build()
@@ -145,7 +145,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         given:
         internalBuild()
         withVersionCatalogue()
-        subProject(":distribution:bwc:staged") << """
+        subProject(":distribution:bwc:minor") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -186,7 +186,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         given:
         internalBuild()
 
-        subProject(":distribution:bwc:staged") << """
+        subProject(":distribution:bwc:minor") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -230,7 +230,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
 
         setupRestResources([], [])
 
-        file("distribution/bwc/staged/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
+        file("distribution/bwc/minor/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
         "one":
           - do:
               do_.some.key_to_replace:

build-tools-internal/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle

@@ -10,9 +10,11 @@
 rootProject.name = "root"
 
 include ":distribution:bwc:bugfix"
+include ":distribution:bwc:bugfix2"
 include ":distribution:bwc:minor"
 include ":distribution:bwc:major"
 include ":distribution:bwc:staged"
+include ":distribution:bwc:maintenance"
 include ":distribution:archives:darwin-tar"
 include ":distribution:archives:oss-darwin-tar"
 include ":distribution:archives:darwin-aarch64-tar"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java

@@ -21,14 +21,15 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.TreeSet;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static java.util.Collections.reverseOrder;
 import static java.util.Collections.unmodifiableList;
+import static java.util.Comparator.comparing;
 
 /**
  * A container for elasticsearch supported version information used in BWC testing.
@@ -73,11 +74,11 @@ public class BwcVersions implements Serializable {
     private final transient List<Version> versions;
     private final Map<Version, UnreleasedVersionInfo> unreleased;
 
-    public BwcVersions(List<String> versionLines) {
-        this(versionLines, Version.fromString(VersionProperties.getElasticsearch()));
+    public BwcVersions(List<String> versionLines, List<String> developmentBranches) {
+        this(versionLines, Version.fromString(VersionProperties.getElasticsearch()), developmentBranches);
     }
 
-    public BwcVersions(Version currentVersionProperty, List<Version> allVersions) {
+    public BwcVersions(Version currentVersionProperty, List<Version> allVersions, List<String> developmentBranches) {
         if (allVersions.isEmpty()) {
             throw new IllegalArgumentException("Could not parse any versions");
         }
@@ -86,12 +87,12 @@ public BwcVersions(Version currentVersionProperty, List<Version> allVersions) {
         this.currentVersion = allVersions.get(allVersions.size() - 1);
         assertCurrentVersionMatchesParsed(currentVersionProperty);
 
-        this.unreleased = computeUnreleased();
+        this.unreleased = computeUnreleased(developmentBranches);
     }
 
     // Visible for testing
-    BwcVersions(List<String> versionLines, Version currentVersionProperty) {
-        this(currentVersionProperty, parseVersionLines(versionLines));
+    BwcVersions(List<String> versionLines, Version currentVersionProperty, List<String> developmentBranches) {
+        this(currentVersionProperty, parseVersionLines(versionLines), developmentBranches);
     }
 
     private static List<Version> parseVersionLines(List<String> versionLines) {
@@ -126,58 +127,77 @@ public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
         getUnreleased().stream().filter(version -> version.equals(currentVersion) == false).map(unreleased::get).forEach(consumer);
     }
 
-    private String getBranchFor(Version version) {
-        if (version.equals(currentVersion)) {
-            // Just assume the current branch is 'main'. It's actually not important, we never check out the current branch.
-            return "main";
-        } else {
+    private String getBranchFor(Version version, List<String> developmentBranches) {
+        // If the current version matches a specific feature freeze branch, use that
+        if (developmentBranches.contains(version.getMajor() + "." + version.getMinor())) {
             return version.getMajor() + "." + version.getMinor();
+        } else if (developmentBranches.contains(version.getMajor() + ".x")) { // Otherwise if an n.x branch exists and we are that major
+            return version.getMajor() + ".x";
+        } else { // otherwise we're the main branch
+            return "main";
         }
     }
 
-    private Map<Version, UnreleasedVersionInfo> computeUnreleased() {
-        Set<Version> unreleased = new TreeSet<>();
-        // The current version is being worked, is always unreleased
-        unreleased.add(currentVersion);
-        // Recurse for all unreleased versions starting from the current version
-        addUnreleased(unreleased, currentVersion, 0);
+    private Map<Version, UnreleasedVersionInfo> computeUnreleased(List<String> developmentBranches) {
+        Map<Version, UnreleasedVersionInfo> result = new TreeMap<>();
 
-        // Grab the latest version from the previous major if necessary as well, this is going to be a maintenance release
-        Version maintenance = versions.stream()
-            .filter(v -> v.getMajor() == currentVersion.getMajor() - 1)
-            .max(Comparator.naturalOrder())
-            .orElseThrow();
-        // This is considered the maintenance release only if we haven't yet encountered it
-        boolean hasMaintenanceRelease = unreleased.add(maintenance);
+        // The current version is always in development
+        String currentBranch = getBranchFor(currentVersion, developmentBranches);
+        result.put(currentVersion, new UnreleasedVersionInfo(currentVersion, currentBranch, ":distribution"));
+
+        // Check for an n.x branch as well
+        if (currentBranch.equals("main") && developmentBranches.stream().anyMatch(s -> s.endsWith(".x"))) {
+            // This should correspond to the latest new minor
+            Version version = versions.stream()
+                .sorted(Comparator.reverseOrder())
+                .filter(v -> v.getMajor() == (currentVersion.getMajor() - 1) && v.getRevision() == 0)
+                .findFirst()
+                .orElseThrow(() -> new IllegalStateException("Unable to determine development version for branch"));
+            String branch = getBranchFor(version, developmentBranches);
+            assert branch.equals(currentVersion.getMajor() - 1 + ".x") : "Expected branch does not match development branch";
+
+            result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:minor"));
+        }
 
-        List<Version> unreleasedList = unreleased.stream().sorted(Comparator.reverseOrder()).toList();
-        Map<Version, UnreleasedVersionInfo> result = new TreeMap<>();
-        boolean newMinor = false;
-        for (int i = 0; i < unreleasedList.size(); i++) {
-            Version esVersion = unreleasedList.get(i);
-            // This is either a new minor or staged release
-            if (currentVersion.equals(esVersion)) {
-                result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution"));
-            } else if (esVersion.getRevision() == 0) {
-                // If there are two upcoming unreleased minors then this one is the new minor
-                if (newMinor == false && unreleasedList.get(i + 1).getRevision() == 0) {
-                    result.put(esVersion, new UnreleasedVersionInfo(esVersion, esVersion.getMajor() + ".x", ":distribution:bwc:minor"));
-                    newMinor = true;
-                } else if (newMinor == false
-                    && unreleasedList.stream().filter(v -> v.getMajor() == esVersion.getMajor() && v.getRevision() == 0).count() == 1) {
-                        // This is the only unreleased new minor which means we've not yet staged it for release
-                        result.put(esVersion, new UnreleasedVersionInfo(esVersion, esVersion.getMajor() + ".x", ":distribution:bwc:minor"));
-                        newMinor = true;
-                    } else {
-                        result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:staged"));
-                    }
-            } else {
-                // If this is the oldest unreleased version and we have a maintenance release
-                if (i == unreleasedList.size() - 1 && hasMaintenanceRelease) {
-                    result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:maintenance"));
-                } else {
-                    result.put(esVersion, new UnreleasedVersionInfo(esVersion, getBranchFor(esVersion), ":distribution:bwc:bugfix"));
-                }
+        // Now handle all the feature freeze branches
+        List<String> featureFreezeBranches = developmentBranches.stream()
+            .filter(b -> Pattern.matches("[0-9]+\\.[0-9]+", b))
+            .sorted(reverseOrder(comparing(s -> Version.fromString(s, Version.Mode.RELAXED))))
+            .toList();
+
+        boolean existingBugfix = false;
+        for (int i = 0; i < featureFreezeBranches.size(); i++) {
+            String branch = featureFreezeBranches.get(i);
+            Version version = versions.stream()
+                .sorted(Comparator.reverseOrder())
+                .filter(v -> v.toString().startsWith(branch))
+                .findFirst()
+                .orElse(null);
+
+            // If we don't know about this version we can ignore it
+            if (version == null) {
+                continue;
+            }
+
+            // If this is the current version we can ignore as we've already handled it
+            if (version.equals(currentVersion)) {
+                continue;
+            }
+
+            // We only maintain compatibility back one major so ignore anything older
+            if (currentVersion.getMajor() - version.getMajor() > 1) {
+                continue;
+            }
+
+            // This is the maintenance version
+            if (i == featureFreezeBranches.size() - 1) {
+                result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:maintenance"));
+            } else if (version.getRevision() == 0) { // This is the next staged minor
+                result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:staged"));
+            } else { // This is a bugfix
+                String project = existingBugfix ? "bugfix2" : "bugfix";
+                result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project));
+                existingBugfix = true;
             }
         }