Merge branch 'main' into poc/expression_score_value

javanna · Sep 21, 2023 · fdb7253 · fdb7253
2 parents bda65ce + 75b6f96
commit fdb7253
Show file tree

Hide file tree

Showing 376 changed files with 5,892 additions and 2,523 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -74,3 +74,8 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
   SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk)
   export SNYK_TOKEN
 fi
+
+if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then
+  # Run in the background, while the job continues
+  nohup .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &
+fi
diff --git a/.buildkite/scripts/setup-monitoring.sh b/.buildkite/scripts/setup-monitoring.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ELASTIC_AGENT_URL=$(vault read -field=url secret/ci/elastic-elasticsearch/elastic-agent-token)
+ELASTIC_AGENT_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/elastic-agent-token)
+
+if [[ ! -d /opt/elastic-agent ]]; then
+  sudo mkdir /opt/elastic-agent
+  sudo chown -R buildkite-agent:buildkite-agent /opt/elastic-agent
+  cd /opt/elastic-agent
+
+  archive=elastic-agent-8.10.1-linux-x86_64.tar.gz
+  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+    archive=elastic-agent-8.10.1-linux-arm64.tar.gz
+  fi
+
+  curl -L -O "https://artifacts.elastic.co/downloads/beats/elastic-agent/$archive"
+
+  tar xzf "$archive" --directory=. --strip-components=1
+fi
+
+cd /opt/elastic-agent
+sudo ./elastic-agent install -f --url="$ELASTIC_AGENT_URL" --enrollment-token="$ELASTIC_AGENT_TOKEN"
diff --git a/...rks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java b/...rks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
@@ -152,7 +152,7 @@ public void setUp() throws Exception {
             .metadata(metadata)
             .routingTable(routingTable)
             .nodes(nb)
-            .compatibilityVersions(compatibilityVersions)
+            .nodeIdsToCompatibilityVersions(compatibilityVersions)
             .build();
     }
 

diff --git a/...s-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/...s-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
@@ -67,6 +67,7 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
         map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");

diff --git a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml
@@ -32,7 +32,7 @@
 
   <!-- Intentionally have multi line string for a bulk request, otherwise this needs to fallback to string concatenation  -->
   <suppress files="modules[/\\]data-streams[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]datastreams[/\\]TsdbDataStreamRestIT.java" checks="LineLength" />
-  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
+  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
 
   <!-- Gradle requires inputs to be seriablizable -->
   <suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

diff --git a/docs/changelog/99091.yaml b/docs/changelog/99091.yaml
@@ -0,0 +1,5 @@
+pr: 99091
+summary: Add flamegraph API
+area: Application
+type: enhancement
+issues: []
diff --git a/docs/changelog/99193.yaml b/docs/changelog/99193.yaml
@@ -0,0 +1,5 @@
+pr: 99193
+summary: Wait for cluster state in recovery
+area: Recovery
+type: enhancement
+issues: []
diff --git a/docs/changelog/99278.yaml b/docs/changelog/99278.yaml
@@ -0,0 +1,5 @@
+pr: 99278
+summary: Support rotatating the JWT shared secret
+area: Security
+type: enhancement
+issues: []
diff --git a/docs/changelog/99432.yaml b/docs/changelog/99432.yaml
@@ -0,0 +1,5 @@
+pr: 99432
+summary: "ESQL: Enable arithmetics for durations and periods"
+area: ES|QL
+type: enhancement
+issues: [99293]
diff --git a/docs/changelog/99474.yaml b/docs/changelog/99474.yaml
@@ -0,0 +1,5 @@
+pr: 99474
+summary: Add `java.net.NetPermission` to APM module's permissions
+area: Infra/Core
+type: bug
+issues: []
diff --git a/docs/changelog/99555.yaml b/docs/changelog/99555.yaml
@@ -0,0 +1,5 @@
+pr: 99555
+summary: Use mappings version to retrieve system index mappings at creation time
+area: Infra/Core
+type: enhancement
+issues: []
diff --git a/docs/changelog/99588.yaml b/docs/changelog/99588.yaml
@@ -0,0 +1,6 @@
+pr: 99588
+summary: Make ESQL more resilient to non-indexed fields
+area: ES|QL
+type: bug
+issues:
+ - 99506
diff --git a/docs/changelog/99627.yaml b/docs/changelog/99627.yaml
@@ -0,0 +1,5 @@
+pr: 99627
+summary: Fix thread context in `getRepositoryData`
+area: Snapshot/Restore
+type: bug
+issues: []
diff --git a/docs/changelog/99641.yaml b/docs/changelog/99641.yaml
@@ -0,0 +1,5 @@
+pr: 99641
+summary: Chunk the cluster allocation explain response
+area: Network
+type: enhancement
+issues: [97803]
diff --git a/docs/changelog/99644.yaml b/docs/changelog/99644.yaml
@@ -0,0 +1,6 @@
+pr: 99644
+summary: Add links to docs from failing bootstrap checks
+area: Infra/Node Lifecycle
+type: enhancement
+issues: [99614]
+
diff --git a/docs/changelog/99655.yaml b/docs/changelog/99655.yaml
@@ -0,0 +1,5 @@
+pr: 99655
+summary: "[Profiling] Allow to wait until resources created"
+area: Application
+type: enhancement
+issues: []
diff --git a/docs/changelog/99677.yaml b/docs/changelog/99677.yaml
@@ -0,0 +1,5 @@
+pr: 99677
+summary: Using 1 MB chunks for elser model storage
+area: Machine Learning
+type: bug
+issues: [ ]
diff --git a/docs/changelog/99682.yaml b/docs/changelog/99682.yaml
@@ -0,0 +1,5 @@
+pr: 99682
+summary: Increase the max vector dims to 4096
+area: Vector Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/99685.yaml b/docs/changelog/99685.yaml
@@ -0,0 +1,5 @@
+pr: 99685
+summary: Fix `advanceExact` for doc values from sources
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/99717.yaml b/docs/changelog/99717.yaml
@@ -0,0 +1,5 @@
+pr: 99717
+summary: Treating watcher webhook response header names as case-insensitive
+area: Watcher
+type: bug
+issues: []
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
@@ -1142,6 +1142,7 @@ You cannot specify both `reindex.ssl.certificate_authorities` and
 
 `reindex.ssl.truststore.password`::
 The password to the truststore (`reindex.ssl.truststore.path`).
+deprecated:[7.17.0] Prefer `reindex.ssl.truststore.secure_password` instead.
 This setting cannot be used with `reindex.ssl.truststore.secure_password`.
 
 `reindex.ssl.truststore.secure_password` (<<secure-settings,Secure>>)::
@@ -1175,6 +1176,7 @@ You cannot specify both `reindex.ssl.key` and `reindex.ssl.keystore.path`.
 `reindex.ssl.key_passphrase`::
 Specifies the passphrase to decrypt the PEM encoded private key
 (`reindex.ssl.key`) if it is encrypted.
+deprecated:[7.17.0] Prefer `reindex.ssl.secure_key_passphrase` instead.
 Cannot be used with `reindex.ssl.secure_key_passphrase`.
 
 `reindex.ssl.secure_key_passphrase` (<<secure-settings,Secure>>)::
@@ -1194,17 +1196,19 @@ If the keystore path ends in ".p12", ".pfx" or "pkcs12", this setting defaults
 to `PKCS12`. Otherwise, it defaults to `jks`.
 
 `reindex.ssl.keystore.password`::
-The password to the keystore (`reindex.ssl.keystore.path`). This setting cannot be used
-with `reindex.ssl.keystore.secure_password`.
+The password to the keystore (`reindex.ssl.keystore.path`).
+deprecated:[7.17.0] Prefer `reindex.ssl.keystore.secure_password` instead.
+This setting cannot be used with `reindex.ssl.keystore.secure_password`.
 
 `reindex.ssl.keystore.secure_password` (<<secure-settings,Secure>>)::
 The password to the keystore (`reindex.ssl.keystore.path`).
 This setting cannot be used with `reindex.ssl.keystore.password`.
 
 `reindex.ssl.keystore.key_password`::
 The password for the key in the keystore (`reindex.ssl.keystore.path`).
-Defaults to the keystore password. This setting cannot be used with
-`reindex.ssl.keystore.secure_key_password`.
+Defaults to the keystore password.
+deprecated:[7.17.0] Prefer `reindex.ssl.keystore.secure_key_password` instead.
+This setting cannot be used with `reindex.ssl.keystore.secure_key_password`.
 
 `reindex.ssl.keystore.secure_key_password` (<<secure-settings,Secure>>)::
 The password for the key in the keystore (`reindex.ssl.keystore.path`).

diff --git a/docs/reference/esql/functions/date_format.asciidoc b/docs/reference/esql/functions/date_format.asciidoc
@@ -7,5 +7,5 @@ is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used.
 ----
 FROM employees
 | KEEP first_name, last_name, hire_date
-| EVAL hired = DATE_FORMAT(hire_date, "YYYY-MM-dd")
+| EVAL hired = DATE_FORMAT("YYYY-MM-dd", hire_date)
 ----
diff --git a/docs/reference/esql/functions/types/date_extract.asciidoc b/docs/reference/esql/functions/types/date_extract.asciidoc
@@ -1,5 +1,5 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 arg1 | arg2 | result
-datetime | keyword | long
+keyword | datetime | long
 |===
diff --git a/docs/reference/ingest/processors/reroute.asciidoc b/docs/reference/ingest/processors/reroute.asciidoc
@@ -4,8 +4,6 @@
 <titleabbrev>Reroute</titleabbrev>
 ++++
 
-experimental::[]
-
 The `reroute` processor allows to route a document to another target index or data stream.
 It has two main modes:
 

diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc
@@ -62,7 +62,7 @@ In many cases, a brute-force kNN search is not efficient enough. For this
 reason, the `dense_vector` type supports indexing vectors into a specialized
 data structure to support fast kNN retrieval through the <<search-api-knn, `knn` option>> in the search API
 
-Unmapped array fields of float elements with size between 128 and 2048 are dynamically mapped as `dense_vector` with a default similariy of `cosine`.
+Unmapped array fields of float elements with size between 128 and 4096 are dynamically mapped as `dense_vector` with a default similariy of `cosine`.
 You can override the default similarity by explicitly mapping the field as `dense_vector` with the desired similarity.
 
 Indexing is enabled by default for dense vector fields.
@@ -132,7 +132,7 @@ integer values between -128 to 127, inclusive for both indexing and searching.
 
 `dims`::
 (Optional, integer)
-Number of vector dimensions. Can't exceed `2048`. If `dims` is not specified,
+Number of vector dimensions. Can't exceed `4096`. If `dims` is not specified,
 it will be set to the length of the first vector added to the field.
 
 `index`::

diff --git a/docs/reference/settings/common-defs.asciidoc b/docs/reference/settings/common-defs.asciidoc
@@ -57,20 +57,21 @@ end::ssl-key-pem[]
 
 tag::ssl-key-passphrase[]
 The passphrase that is used to decrypt the private key. Since the key might not
-be encrypted, this value is optional.
+be encrypted, this value is optional. deprecated:[7.17.0] Prefer `ssl.secure_key_passphrase` instead.
 +
 You cannot use this setting and `ssl.secure_key_passphrase` at the same time.
 end::ssl-key-passphrase[]
 
 tag::ssl-keystore-key-password[]
 The password for the key in the keystore. The default is the keystore password.
+deprecated:[7.17.0] Prefer `ssl.keystore.secure_key_password` instead.
 +
 You cannot use this setting and `ssl.keystore.secure_password` at the same time.
 //TBD: You cannot use this setting and `ssl.keystore.secure_key_password` at the same time.
 end::ssl-keystore-key-password[]
 
 tag::ssl-keystore-password[]
-The password for the keystore.
+The password for the keystore. deprecated:[7.17.0] Prefer `ssl.keystore.secure_password` instead.
 //TBD: You cannot use this setting and `ssl.keystore.secure_password` at the same time.
 end::ssl-keystore-password[]
 
@@ -122,7 +123,7 @@ or `SSLv3`. See <<fips-140-compliance>>.
 end::ssl-supported-protocols[]
 
 tag::ssl-truststore-password[]
-The password for the truststore.
+The password for the truststore. deprecated:[7.17.0] Prefer `ssl.truststore.secure_password` instead.
 +
 You cannot use this setting and `ssl.truststore.secure_password` at the same
 time.
@@ -160,7 +161,7 @@ Authority (CA); has a `hostname` or IP address that matches the names within
 the certificate.
 
 `certificate`::
-Validates the provided certificate and verifies that it's signed by a 
+Validates the provided certificate and verifies that it's signed by a
 trusted authority (CA), but doesn't check the certificate `hostname`.
 
 `none`::
@@ -173,4 +174,4 @@ resolve TLS errors.
 =====
 +
 Defaults to `full`.
-end::ssl-verification-mode-values[]
+end::ssl-verification-mode-values[]