Merge branch '8.x' into fix/analysis-tokenizer-edgeNGram

YeonghyeonKO · Sep 23, 2024 · cedec13 · cedec13
2 parents 36c2cf1 + f95f292
commit cedec13
Show file tree

Hide file tree

Showing 201 changed files with 3,504 additions and 1,024 deletions.
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -168,7 +168,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
             vmParameters = [
               '-ea',
               '-Djava.security.manager=allow',
-              '-Djava.locale.providers=SPI,CLDR',
+              '-Djava.locale.providers=CLDR',
               '-Des.nativelibs.path="' + testLibraryPath + '"',
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',

diff --git a/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
@@ -93,7 +93,7 @@ public void execute(Task t) {
                     mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile());
 
                     // TODO remove once jvm.options are added to test system properties
-                    test.systemProperty("java.locale.providers", "SPI,CLDR");
+                    test.systemProperty("java.locale.providers", "CLDR");
                 }
             });
             test.getJvmArgumentProviders().add(nonInputProperties);

diff --git a/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
@@ -76,11 +76,11 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
     @UpdateForV9    // only use CLDR in v9+
     private static String getLocaleProviders() {
         /*
-         * Specify SPI to load IsoCalendarDataProvider (see #48209), specifying the first day of week as Monday.
-         * When on pre-23, use COMPAT instead to maintain existing date formats as much as we can.
-         * When on JDK 23+, use the default CLDR locale database, as COMPAT was removed in JDK 23.
+         * When on pre-23, use COMPAT instead to maintain existing date formats as much as we can,
+         * and specify SPI to load IsoCalendarDataProvider (see #48209), specifying the first day of week as Monday.
+         * When on JDK 23+, use the default CLDR locale database for everything, as COMPAT was removed in JDK 23.
          */
-        return Runtime.version().feature() >= 23 ? "SPI,CLDR" : "SPI,COMPAT";
+        return Runtime.version().feature() >= 23 ? "CLDR" : "SPI,COMPAT";
     }
 
     /*

diff --git a/docs/changelog/112645.yaml b/docs/changelog/112645.yaml
@@ -0,0 +1,6 @@
+pr: 112645
+summary: Add support for multi-value dimensions
+area: Mapping
+type: enhancement
+issues:
+ - 110387
diff --git a/docs/changelog/113103.yaml b/docs/changelog/113103.yaml
@@ -0,0 +1,6 @@
+pr: 113103
+summary: "ESQL: Align year diffing to the rest of the units in DATE_DIFF: chronological"
+area: ES|QL
+type: bug
+issues:
+ - 112482
diff --git a/docs/changelog/113172.yaml b/docs/changelog/113172.yaml
@@ -0,0 +1,6 @@
+pr: 113172
+summary: "[ESQL] Add finish() elapsed time to aggregation profiling times"
+area: ES|QL
+type: enhancement
+issues:
+ - 112950
diff --git a/docs/changelog/113280.yaml b/docs/changelog/113280.yaml
@@ -0,0 +1,5 @@
+pr: 113280
+summary: Warn for model load failures if they have a status code <500
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/reference/esql/functions/description/qstr.asciidoc b/docs/reference/esql/functions/description/qstr.asciidoc
diff --git a/docs/reference/esql/functions/examples/date_diff.asciidoc b/docs/reference/esql/functions/examples/date_diff.asciidoc
diff --git a/docs/reference/esql/functions/examples/qstr.asciidoc b/docs/reference/esql/functions/examples/qstr.asciidoc
diff --git a/docs/reference/esql/functions/kibana/definition/date_diff.json b/docs/reference/esql/functions/kibana/definition/date_diff.json
diff --git a/docs/reference/esql/functions/kibana/definition/qstr.json b/docs/reference/esql/functions/kibana/definition/qstr.json
diff --git a/docs/reference/esql/functions/kibana/docs/mv_avg.md b/docs/reference/esql/functions/kibana/docs/mv_avg.md
diff --git a/docs/reference/esql/functions/kibana/docs/mv_sum.md b/docs/reference/esql/functions/kibana/docs/mv_sum.md
diff --git a/docs/reference/esql/functions/kibana/docs/qstr.md b/docs/reference/esql/functions/kibana/docs/qstr.md
diff --git a/docs/reference/esql/functions/layout/qstr.asciidoc b/docs/reference/esql/functions/layout/qstr.asciidoc
diff --git a/docs/reference/esql/functions/parameters/qstr.asciidoc b/docs/reference/esql/functions/parameters/qstr.asciidoc
diff --git a/docs/reference/esql/functions/signature/qstr.svg b/docs/reference/esql/functions/signature/qstr.svg
diff --git a/docs/reference/esql/functions/types/qstr.asciidoc b/docs/reference/esql/functions/types/qstr.asciidoc
diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
@@ -163,7 +163,6 @@ index setting limits the number of dimensions in an index.
 Dimension fields have the following constraints:
 
 * The `doc_values` and `index` mapping parameters must be `true`.
-* Field values cannot be an <<array,array or multi-value>>.
 // end::dimension[]
 * Dimension values are used to identify a document’s time series. If dimension values are altered in any way during indexing, the document will be stored as belonging to different from intended time series. As a result there are additional constraints:
 ** The field cannot use a <<normalizer,`normalizer`>>.

diff --git a/docs/reference/query-dsl/sparse-vector-query.asciidoc b/docs/reference/query-dsl/sparse-vector-query.asciidoc
@@ -104,7 +104,7 @@ Default: `5`.
 `tokens_weight_threshold`::
 (Optional, float)
 preview:[]
-Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc
@@ -68,7 +68,7 @@ Default: `5`.
 `tokens_weight_threshold`::
 (Optional, float)
 preview:[]
-Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

diff --git a/docs/reference/query-dsl/weighted-tokens-query.asciidoc b/docs/reference/query-dsl/weighted-tokens-query.asciidoc
@@ -58,7 +58,7 @@ This value must between 1 and 100.
 Default: `5`.
 
 `tokens_weight_threshold`::
-(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned.
+(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned.
 This value must be between 0 and 1.
 Default: `0.4`.
 

diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc
@@ -44,6 +44,14 @@ To work around this issue, you have a number of options:
 <<esql-kibana-enable,disable ES|QL queries in {kib}>>
 ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts.
 
+* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or
+partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved.
+There is no workaround and the only option to is to upgrade to 8.15.2 when released.
++
+If you use synthetic source then you may be affected by this bug if the following is true:
+** If you have more fields then the `index.mapping.total_fields.limit` setting allows.
+** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled.
+
 [[breaking-8.15.0]]
 [float]
 === Breaking changes

diff --git a/docs/reference/release-notes/8.15.1.asciidoc b/docs/reference/release-notes/8.15.1.asciidoc
@@ -24,13 +24,20 @@ To work around this issue, you have a number of options:
 <<esql-kibana-enable,disable ES|QL queries in {kib}>>
 ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts.
 
-* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field 
+* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field
 but there is an index segment that does not contain any documents with a dense vector field ({es-pull}112720[#112720]). Workarounds:
 ** If the affected index already contains documents with a dense vector field, force merge the index to a single segment.
 ** If the affected index does not already contain documents with a dense vector field, index a document with a dense vector field
 and then force merge to a single segment.
 ** If the affected index's `dense_vector` fields are unused, reindex without the `dense_vector` fields.
 
+* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or
+partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved.
+There is no workaround and the only option to is to upgrade to 8.15.2 when released.
++
+If you use synthetic source then you may be affected by this bug if the following is true:
+** If you have more fields then the `index.mapping.total_fields.limit` setting allows.
+** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled.
 
 [[bug-8.15.1]]
 [float]

diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml
@@ -1230,3 +1230,83 @@ non string dimension fields:
   - match: { .$idx0name.mappings.properties.attributes.properties.double.time_series_dimension: true }
   - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' }
   - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.time_series_dimension: true }
+
+---
+multi value dimensions:
+  - requires:
+      cluster_features: ["routing.multi_value_routing_path"]
+      reason: support for multi-value dimensions
+
+  - do:
+      allowed_warnings:
+        - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-dynamic-template
+        body:
+          index_patterns: [k9s*]
+          data_stream: {}
+          template:
+            settings:
+              index:
+                number_of_shards: 1
+                mode: time_series
+                time_series:
+                  start_time: 2023-08-31T13:03:08.138Z
+
+            mappings:
+              properties:
+                attributes:
+                  type: passthrough
+                  dynamic: true
+                  time_series_dimension: true
+                  priority: 1
+              dynamic_templates:
+                - counter_metric:
+                    mapping:
+                      type: integer
+                      time_series_metric: counter
+
+  - do:
+      bulk:
+        index: k9s
+        refresh: true
+        body:
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes": { "dim1": ["a" , "b"], "dim2": [1, 2] } }'
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "20", "attributes": { "dim1": ["b" , "a"], "dim2": [1, 2] } }'
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "20", "attributes": { "dim1": ["c" , "b"], "dim2": [1, 2] } }'
+  - is_false: errors
+
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+
+  - length: { aggregations.tsids.buckets: 3 } # only the order of the dim1 attribute is different, yet we expect to have two distinct time series
+
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+          aggs:
+            dims:
+              terms:
+                field: dim1
+                order:
+                  _key: asc
+
+  - length: { aggregations.dims.buckets: 3 }
+  - match: { aggregations.dims.buckets.0.key: a }
+  - match: { aggregations.dims.buckets.0.doc_count: 2 }
+  - match: { aggregations.dims.buckets.1.key: b }
+  - match: { aggregations.dims.buckets.1.doc_count: 3 }
+  - match: { aggregations.dims.buckets.2.key: c }
+  - match: { aggregations.dims.buckets.2.doc_count: 1 }
diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle
@@ -55,13 +55,15 @@ tasks.named("yamlRestTest").configure {
   systemProperty 'tests.rest.blacklist', [
     // for some reason, allowed_warnings on the test isn't working here
     'ingest/30_date_processor/Test date processor with no timezone configured',
+    'ingest/30_date_processor/Test week based date parsing',
   ].join(',')
 }
 
 tasks.named("yamlRestTestV7CompatTest").configure {
   systemProperty 'tests.rest.blacklist', [
     // for some reason, allowed_warnings on the test isn't working here
     'ingest/30_date_processor/Test date processor with no timezone configured',
+    'ingest/30_date_processor/Test week based date parsing',
   ].join(',')
 }