diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle index 221dccae91322..d4bb0c1189e8d 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle @@ -168,7 +168,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { vmParameters = [ '-ea', '-Djava.security.manager=allow', - '-Djava.locale.providers=SPI,CLDR', + '-Djava.locale.providers=CLDR', '-Des.nativelibs.path="' + testLibraryPath + '"', // TODO: only open these for mockito when it is modularized '--add-opens=java.base/java.security.cert=ALL-UNNAMED', diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java index 96cfa24c4c7d9..19ab49a851907 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java @@ -93,7 +93,7 @@ public void execute(Task t) { mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile()); // TODO remove once jvm.options are added to test system properties - test.systemProperty("java.locale.providers", "SPI,CLDR"); + test.systemProperty("java.locale.providers", "CLDR"); } }); test.getJvmArgumentProviders().add(nonInputProperties); diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 784ecb18f8bd9..c279da0465f2a 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -76,11 +76,11 @@ static List systemJvmOptions(Settings nodeSettings, final Map= 23 ? "SPI,CLDR" : "SPI,COMPAT"; + return Runtime.version().feature() >= 23 ? "CLDR" : "SPI,COMPAT"; } /* diff --git a/docs/changelog/112645.yaml b/docs/changelog/112645.yaml new file mode 100644 index 0000000000000..cf4ef4609a1f3 --- /dev/null +++ b/docs/changelog/112645.yaml @@ -0,0 +1,6 @@ +pr: 112645 +summary: Add support for multi-value dimensions +area: Mapping +type: enhancement +issues: + - 110387 diff --git a/docs/changelog/113103.yaml b/docs/changelog/113103.yaml new file mode 100644 index 0000000000000..2ed98e0907bae --- /dev/null +++ b/docs/changelog/113103.yaml @@ -0,0 +1,6 @@ +pr: 113103 +summary: "ESQL: Align year diffing to the rest of the units in DATE_DIFF: chronological" +area: ES|QL +type: bug +issues: + - 112482 diff --git a/docs/changelog/113172.yaml b/docs/changelog/113172.yaml new file mode 100644 index 0000000000000..2d03196b0cfbd --- /dev/null +++ b/docs/changelog/113172.yaml @@ -0,0 +1,6 @@ +pr: 113172 +summary: "[ESQL] Add finish() elapsed time to aggregation profiling times" +area: ES|QL +type: enhancement +issues: + - 112950 diff --git a/docs/changelog/113280.yaml b/docs/changelog/113280.yaml new file mode 100644 index 0000000000000..1d8de0d87dd0d --- /dev/null +++ b/docs/changelog/113280.yaml @@ -0,0 +1,5 @@ +pr: 113280 +summary: Warn for model load failures if they have a status code <500 +area: Machine Learning +type: bug +issues: [] diff --git a/docs/reference/esql/functions/description/qstr.asciidoc b/docs/reference/esql/functions/description/qstr.asciidoc new file mode 100644 index 0000000000000..5ce9316405ad2 --- /dev/null +++ b/docs/reference/esql/functions/description/qstr.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Performs a query string query. Returns true if the provided query string matches the row. diff --git a/docs/reference/esql/functions/examples/date_diff.asciidoc b/docs/reference/esql/functions/examples/date_diff.asciidoc index f85bdf480c1c3..f75add7b80501 100644 --- a/docs/reference/esql/functions/examples/date_diff.asciidoc +++ b/docs/reference/esql/functions/examples/date_diff.asciidoc @@ -1,6 +1,6 @@ // This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. -*Example* +*Examples* [source.merge.styled,esql] ---- @@ -10,4 +10,15 @@ include::{esql-specs}/date.csv-spec[tag=docsDateDiff] |=== include::{esql-specs}/date.csv-spec[tag=docsDateDiff-result] |=== +When subtracting in calendar units - like year, month a.s.o. - only the fully elapsed units are counted. +To avoid this and obtain also remainders, simply switch to the next smaller unit and do the date math accordingly. + +[source.merge.styled,esql] +---- +include::{esql-specs}/date.csv-spec[tag=evalDateDiffYearForDocs] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/date.csv-spec[tag=evalDateDiffYearForDocs-result] +|=== diff --git a/docs/reference/esql/functions/examples/qstr.asciidoc b/docs/reference/esql/functions/examples/qstr.asciidoc new file mode 100644 index 0000000000000..003373c84c029 --- /dev/null +++ b/docs/reference/esql/functions/examples/qstr.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/qstr-function.csv-spec[tag=qstr-with-field] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/qstr-function.csv-spec[tag=qstr-with-field-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/date_diff.json b/docs/reference/esql/functions/kibana/definition/date_diff.json index d6589f041075d..f4c4de53f72a3 100644 --- a/docs/reference/esql/functions/kibana/definition/date_diff.json +++ b/docs/reference/esql/functions/kibana/definition/date_diff.json @@ -54,6 +54,8 @@ } ], "examples" : [ - "ROW date1 = TO_DATETIME(\"2023-12-02T11:00:00.000Z\"), date2 = TO_DATETIME(\"2023-12-02T11:00:00.001Z\")\n| EVAL dd_ms = DATE_DIFF(\"microseconds\", date1, date2)" - ] + "ROW date1 = TO_DATETIME(\"2023-12-02T11:00:00.000Z\"), date2 = TO_DATETIME(\"2023-12-02T11:00:00.001Z\")\n| EVAL dd_ms = DATE_DIFF(\"microseconds\", date1, date2)", + "ROW end_23=\"2023-12-31T23:59:59.999Z\"::DATETIME,\n start_24=\"2024-01-01T00:00:00.000Z\"::DATETIME,\n end_24=\"2024-12-31T23:59:59.999\"::DATETIME\n| EVAL end23_to_start24=DATE_DIFF(\"year\", end_23, start_24)\n| EVAL end23_to_end24=DATE_DIFF(\"year\", end_23, end_24)\n| EVAL start_to_end_24=DATE_DIFF(\"year\", start_24, end_24)" + ], + "preview" : false } diff --git a/docs/reference/esql/functions/kibana/definition/qstr.json b/docs/reference/esql/functions/kibana/definition/qstr.json new file mode 100644 index 0000000000000..dfa3dfd3818ad --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/qstr.json @@ -0,0 +1,36 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "qstr", + "description" : "Performs a query string query. Returns true if the provided query string matches the row.", + "signatures" : [ + { + "params" : [ + { + "name" : "query", + "type" : "keyword", + "optional" : false, + "description" : "Query string in Lucene query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "query", + "type" : "text", + "optional" : false, + "description" : "Query string in Lucene query string format." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], + "examples" : [ + "from books \n| where qstr(\"author: Faulkner\")\n| keep book_no, author \n| sort book_no \n| limit 5;" + ], + "preview" : true +} diff --git a/docs/reference/esql/functions/kibana/docs/mv_avg.md b/docs/reference/esql/functions/kibana/docs/mv_avg.md index c3d7e5423f724..c5163f36129bf 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_avg.md +++ b/docs/reference/esql/functions/kibana/docs/mv_avg.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_AVG -Converts a multivalued field into a single valued field containing the average of all of the values. +Converts a multivalued field into a single valued field containing the average of all the values. ``` ROW a=[3, 5, 1, 6] diff --git a/docs/reference/esql/functions/kibana/docs/mv_sum.md b/docs/reference/esql/functions/kibana/docs/mv_sum.md index 16285d3c7229b..987017b34b743 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_sum.md +++ b/docs/reference/esql/functions/kibana/docs/mv_sum.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_SUM -Converts a multivalued field into a single valued field containing the sum of all of the values. +Converts a multivalued field into a single valued field containing the sum of all the values. ``` ROW a=[3, 5, 6] diff --git a/docs/reference/esql/functions/kibana/docs/qstr.md b/docs/reference/esql/functions/kibana/docs/qstr.md new file mode 100644 index 0000000000000..37b5777623185 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/qstr.md @@ -0,0 +1,14 @@ + + +### QSTR +Performs a query string query. Returns true if the provided query string matches the row. + +``` +from books +| where qstr("author: Faulkner") +| keep book_no, author +| sort book_no +| limit 5; +``` diff --git a/docs/reference/esql/functions/layout/qstr.asciidoc b/docs/reference/esql/functions/layout/qstr.asciidoc new file mode 100644 index 0000000000000..715a11089f0d4 --- /dev/null +++ b/docs/reference/esql/functions/layout/qstr.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-qstr]] +=== `QSTR` + +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/qstr.svg[Embedded,opts=inline] + +include::../parameters/qstr.asciidoc[] +include::../description/qstr.asciidoc[] +include::../types/qstr.asciidoc[] +include::../examples/qstr.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/qstr.asciidoc b/docs/reference/esql/functions/parameters/qstr.asciidoc new file mode 100644 index 0000000000000..e51096084f2f3 --- /dev/null +++ b/docs/reference/esql/functions/parameters/qstr.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`query`:: +Query string in Lucene query string format. diff --git a/docs/reference/esql/functions/signature/qstr.svg b/docs/reference/esql/functions/signature/qstr.svg new file mode 100644 index 0000000000000..0d3841b071cef --- /dev/null +++ b/docs/reference/esql/functions/signature/qstr.svg @@ -0,0 +1 @@ +QSTR(query) diff --git a/docs/reference/esql/functions/types/qstr.asciidoc b/docs/reference/esql/functions/types/qstr.asciidoc new file mode 100644 index 0000000000000..866a39e925665 --- /dev/null +++ b/docs/reference/esql/functions/types/qstr.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +query | result +keyword | boolean +text | boolean +|=== diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index 59d307c4df0ad..a4be7026dffcd 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -163,7 +163,6 @@ index setting limits the number of dimensions in an index. Dimension fields have the following constraints: * The `doc_values` and `index` mapping parameters must be `true`. -* Field values cannot be an <>. // end::dimension[] * Dimension values are used to identify a document’s time series. If dimension values are altered in any way during indexing, the document will be stored as belonging to different from intended time series. As a result there are additional constraints: ** The field cannot use a <>. diff --git a/docs/reference/query-dsl/sparse-vector-query.asciidoc b/docs/reference/query-dsl/sparse-vector-query.asciidoc index 08dd7ab7f4470..399cf29d4dd12 100644 --- a/docs/reference/query-dsl/sparse-vector-query.asciidoc +++ b/docs/reference/query-dsl/sparse-vector-query.asciidoc @@ -104,7 +104,7 @@ Default: `5`. `tokens_weight_threshold`:: (Optional, float) preview:[] -Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index 8faecad1dbdb9..235a413df686f 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -68,7 +68,7 @@ Default: `5`. `tokens_weight_threshold`:: (Optional, float) preview:[] -Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/query-dsl/weighted-tokens-query.asciidoc b/docs/reference/query-dsl/weighted-tokens-query.asciidoc index d4318665a9778..fb051f4229df6 100644 --- a/docs/reference/query-dsl/weighted-tokens-query.asciidoc +++ b/docs/reference/query-dsl/weighted-tokens-query.asciidoc @@ -58,7 +58,7 @@ This value must between 1 and 100. Default: `5`. `tokens_weight_threshold`:: -(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc index 1496d7846a080..80c86c7079f0c 100644 --- a/docs/reference/release-notes/8.15.0.asciidoc +++ b/docs/reference/release-notes/8.15.0.asciidoc @@ -44,6 +44,14 @@ To work around this issue, you have a number of options: <> ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts. +* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or +partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved. +There is no workaround and the only option to is to upgrade to 8.15.2 when released. ++ +If you use synthetic source then you may be affected by this bug if the following is true: +** If you have more fields then the `index.mapping.total_fields.limit` setting allows. +** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled. + [[breaking-8.15.0]] [float] === Breaking changes diff --git a/docs/reference/release-notes/8.15.1.asciidoc b/docs/reference/release-notes/8.15.1.asciidoc index e3bfaa18b6986..7c48f457e3b4e 100644 --- a/docs/reference/release-notes/8.15.1.asciidoc +++ b/docs/reference/release-notes/8.15.1.asciidoc @@ -24,13 +24,20 @@ To work around this issue, you have a number of options: <> ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts. -* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field +* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field but there is an index segment that does not contain any documents with a dense vector field ({es-pull}112720[#112720]). Workarounds: ** If the affected index already contains documents with a dense vector field, force merge the index to a single segment. ** If the affected index does not already contain documents with a dense vector field, index a document with a dense vector field and then force merge to a single segment. ** If the affected index's `dense_vector` fields are unused, reindex without the `dense_vector` fields. +* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or +partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved. +There is no workaround and the only option to is to upgrade to 8.15.2 when released. ++ +If you use synthetic source then you may be affected by this bug if the following is true: +** If you have more fields then the `index.mapping.total_fields.limit` setting allows. +** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled. [[bug-8.15.1]] [float] diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index d20231a6d6cf2..56f387c016261 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -1230,3 +1230,83 @@ non string dimension fields: - match: { .$idx0name.mappings.properties.attributes.properties.double.time_series_dimension: true } - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' } - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.time_series_dimension: true } + +--- +multi value dimensions: + - requires: + cluster_features: ["routing.multi_value_routing_path"] + reason: support for multi-value dimensions + + - do: + allowed_warnings: + - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" + indices.put_index_template: + name: my-dynamic-template + body: + index_patterns: [k9s*] + data_stream: {} + template: + settings: + index: + number_of_shards: 1 + mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z + + mappings: + properties: + attributes: + type: passthrough + dynamic: true + time_series_dimension: true + priority: 1 + dynamic_templates: + - counter_metric: + mapping: + type: integer + time_series_metric: counter + + - do: + bulk: + index: k9s + refresh: true + body: + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes": { "dim1": ["a" , "b"], "dim2": [1, 2] } }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "20", "attributes": { "dim1": ["b" , "a"], "dim2": [1, 2] } }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "20", "attributes": { "dim1": ["c" , "b"], "dim2": [1, 2] } }' + - is_false: errors + + - do: + search: + index: k9s + body: + size: 0 + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.tsids.buckets: 3 } # only the order of the dim1 attribute is different, yet we expect to have two distinct time series + + - do: + search: + index: k9s + body: + size: 0 + aggs: + dims: + terms: + field: dim1 + order: + _key: asc + + - length: { aggregations.dims.buckets: 3 } + - match: { aggregations.dims.buckets.0.key: a } + - match: { aggregations.dims.buckets.0.doc_count: 2 } + - match: { aggregations.dims.buckets.1.key: b } + - match: { aggregations.dims.buckets.1.doc_count: 3 } + - match: { aggregations.dims.buckets.2.key: c } + - match: { aggregations.dims.buckets.2.doc_count: 1 } diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index bdeed16231aca..94ef35c304f63 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -55,6 +55,7 @@ tasks.named("yamlRestTest").configure { systemProperty 'tests.rest.blacklist', [ // for some reason, allowed_warnings on the test isn't working here 'ingest/30_date_processor/Test date processor with no timezone configured', + 'ingest/30_date_processor/Test week based date parsing', ].join(',') } @@ -62,6 +63,7 @@ tasks.named("yamlRestTestV7CompatTest").configure { systemProperty 'tests.rest.blacklist', [ // for some reason, allowed_warnings on the test isn't working here 'ingest/30_date_processor/Test date processor with no timezone configured', + 'ingest/30_date_processor/Test week based date parsing', ].join(',') } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index 600a56a4fa8e2..554e074bcc7e4 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -75,17 +75,17 @@ public void testParseJavaDefaultYear() { public void testParseWeekBasedYearAndWeek() { String format = "YYYY-ww"; ZoneId timezone = DateUtils.of("Europe/Amsterdam"); - Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ENGLISH); ZonedDateTime dateTime = javaFunction.apply("2020-33"); - assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 8, 10, 0, 0, 0, 0, timezone))); + assertThat(dateTime, equalTo(ZonedDateTime.of(2020, 8, 9, 0, 0, 0, 0, timezone))); } public void testParseWeekBasedYear() { String format = "YYYY"; ZoneId timezone = DateUtils.of("Europe/Amsterdam"); - Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ENGLISH); ZonedDateTime dateTime = javaFunction.apply("2019"); - assertThat(dateTime, equalTo(ZonedDateTime.of(2018, 12, 31, 0, 0, 0, 0, timezone))); + assertThat(dateTime, equalTo(ZonedDateTime.of(2018, 12, 30, 0, 0, 0, 0, timezone))); } public void testParseWeekBasedWithLocale() { diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml index 8508534614c4b..b2710ff05aa00 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -211,6 +211,8 @@ teardown: --- "Test week based date parsing": - do: + allowed_warnings: + - 'Date format [YYYY-ww] contains week-date field specifiers that are changing in JDK 23' indices.create: index: test body: @@ -221,6 +223,8 @@ teardown: format: YYYY-ww - do: + allowed_warnings: + - 'Date format [YYYY-ww] contains week-date field specifiers that are changing in JDK 23' ingest.put_pipeline: id: "my_pipeline" body: > @@ -270,69 +274,3 @@ teardown: id: "1" - match: { _source.date_source_field: "2020-33" } - match: { _source.date_target_field: "2020-08-10T00:00:00.000Z" } - ---- -"Test week based date parsing with locale": - #locale is used when parsing as well on a pipeline. As per US locale, start of the 33rd week 2020 is on 09August2020 (sunday) - - do: - indices.create: - index: test - body: - mappings: - properties: - date_source_field: - type: date - format: YYYY-ww - locale: en-US - - - do: - ingest.put_pipeline: - id: "my_pipeline" - body: > - { - "description": "_description", - "processors": [ - { - "date" : { - "field" : "date_source_field", - "target_field" : "date_target_field", - "formats" : ["YYYY-ww"], - "locale" : "en-US" - } - } - ] - } - - match: { acknowledged: true } - - - do: - ingest.simulate: - id: "my_pipeline" - body: > - { - "docs": [ - { - "_source": { - "date_source_field": "2020-33" - } - } - ] - } - - length: { docs: 1 } - - match: { docs.0.doc._source.date_source_field: "2020-33" } - - match: { docs.0.doc._source.date_target_field: "2020-08-09T00:00:00.000Z" } - - length: { docs.0.doc._ingest: 1 } - - is_true: docs.0.doc._ingest.timestamp - - - do: - index: - index: test - id: "1" - pipeline: "my_pipeline" - body: {date_source_field: "2020-33"} - - - do: - get: - index: test - id: "1" - - match: { _source.date_source_field: "2020-33" } - - match: { _source.date_target_field: "2020-08-09T00:00:00.000Z" } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java index 3c9e684ef4279..021ce09e0ed8e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java @@ -46,7 +46,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception assert msg instanceof HttpObject; if (msg instanceof HttpRequest request) { var preReq = HttpHeadersAuthenticatorUtils.asHttpPreRequest(request); - aggregating = decider.test(preReq) && IGNORE_TEST.test(preReq); + aggregating = (decider.test(preReq) && IGNORE_TEST.test(preReq)) || request.decoderResult().isFailure(); } if (aggregating || msg instanceof FullHttpRequest) { super.channelRead(ctx, msg); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java index b04da46a2d7d7..a1aa211814520 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java @@ -17,6 +17,7 @@ import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.QueryStringDecoder; import io.netty.handler.codec.http.cookie.Cookie; import io.netty.handler.codec.http.cookie.ServerCookieDecoder; import io.netty.handler.codec.http.cookie.ServerCookieEncoder; @@ -48,6 +49,7 @@ public class Netty4HttpRequest implements HttpRequest { private final Exception inboundException; private final boolean pooled; private final int sequence; + private final QueryStringDecoder queryStringDecoder; Netty4HttpRequest(int sequence, io.netty.handler.codec.http.HttpRequest request, Netty4HttpRequestBodyStream contentStream) { this( @@ -94,6 +96,7 @@ private Netty4HttpRequest( this.pooled = pooled; this.released = released; this.inboundException = inboundException; + this.queryStringDecoder = new QueryStringDecoder(request.uri()); } @Override @@ -106,6 +109,11 @@ public String uri() { return request.uri(); } + @Override + public String rawPath() { + return queryStringDecoder.rawPath(); + } + @Override public HttpBody body() { assert released.get() == false; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index c6e7fa3517771..5ed3d81392951 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -375,9 +375,8 @@ protected HttpMessage createMessage(String[] initialLine) throws Exception { final HttpObjectAggregator aggregator = new Netty4HttpAggregator( handlingSettings.maxContentLength(), httpPreRequest -> enabled.get() == false - || (httpPreRequest.uri().contains("_bulk") == false - || httpPreRequest.uri().contains("_bulk_update") - || httpPreRequest.uri().contains("/_xpack/monitoring/_bulk")) + || ((httpPreRequest.rawPath().endsWith("/_bulk") == false) + || httpPreRequest.rawPath().startsWith("/_xpack/monitoring/_bulk")) ); aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); ch.pipeline() diff --git a/muted-tests.yml b/muted-tests.yml index ef2cb030bfc38..44cb6f631ddff 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -205,6 +205,12 @@ tests: - class: org.elasticsearch.packaging.test.WindowsServiceTests method: test30StartStop issue: https://github.com/elastic/elasticsearch/issues/113160 +- class: org.elasticsearch.packaging.test.WindowsServiceTests + method: test33JavaChanged + issue: https://github.com/elastic/elasticsearch/issues/113177 +- class: org.elasticsearch.datastreams.logsdb.qa.StandardVersusLogsIndexModeRandomDataChallengeRestIT + method: testMatchAllQuery + issue: https://github.com/elastic/elasticsearch/issues/113265 # Examples: # diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java index 08026e0435f33..2b24e53874e51 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java @@ -29,6 +29,12 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0) public class IncrementalBulkRestIT extends HttpSmokeTestCase { + public void testBulkUriMatchingDoesNotMatchBulkCapabilitiesApi() throws IOException { + Request request = new Request("GET", "/_capabilities?method=GET&path=%2F_bulk&capabilities=failure_store_status&pretty"); + Response response = getRestClient().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + } + public void testBulkMissingBody() throws IOException { Request request = new Request(randomBoolean() ? "POST" : "PUT", "/_bulk"); request.setJsonEntity(""); diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java index c652a9992d8fb..322c055de140b 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java @@ -19,7 +19,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -102,22 +101,9 @@ private void runTest(String actionName, String endpoint) throws Exception { private void updateClusterState(Function transformationFn) { final TimeValue timeout = TimeValue.timeValueSeconds(10); - - final AckedRequest ackedRequest = new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return timeout; - } - - @Override - public TimeValue masterNodeTimeout() { - return timeout; - } - }; - PlainActionFuture future = new PlainActionFuture<>(); internalCluster().getAnyMasterNodeInstance(ClusterService.class) - .submitUnbatchedStateUpdateTask("get_mappings_cancellation_test", new AckedClusterStateUpdateTask(ackedRequest, future) { + .submitUnbatchedStateUpdateTask("get_mappings_cancellation_test", new AckedClusterStateUpdateTask(timeout, timeout, future) { @Override public ClusterState execute(ClusterState currentState) throws Exception { return transformationFn.apply(currentState); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 8a27eb3efd219..937c5f19ae5aa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -1269,8 +1269,8 @@ synthetic_source with copy_to and ignored values: refresh: true body: name: "B" - k: ["5", "6"] - long: ["7", "8"] + k: ["55", "66"] + long: ["77", "88"] - do: search: @@ -1289,10 +1289,9 @@ synthetic_source with copy_to and ignored values: - match: hits.hits.1._source: name: "B" - k: ["5", "6"] - long: ["7", "8"] - - match: { hits.hits.1.fields.copy: ["5", "6", "7", "8"] } - + k: ["55", "66"] + long: ["77", "88"] + - match: { hits.hits.1.fields.copy: ["55", "66", "77", "88"] } --- synthetic_source with copy_to field having values in source: @@ -1553,3 +1552,402 @@ synthetic_source with copy_to and invalid values for copy: - match: { error.type: "document_parsing_exception" } - contains: { error.reason: "Copy-to currently only works for value-type fields" } + +--- +synthetic_source with copy_to pointing inside object: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + my_values: + properties: + k: + type: keyword + ignore_above: 1 + copy_to: c.copy + long: + type: long + copy_to: c.copy + c: + properties: + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + my_values: + k: "hello" + long: 100 + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + my_values: + k: ["55", "66"] + long: [77, 88] + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + my_values: + k: "hello" + long: 100 + c: + copy: "zap" + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "c.copy" ] + + - match: + hits.hits.0._source: + name: "A" + my_values: + k: "hello" + long: 100 + - match: + hits.hits.0.fields: + c.copy: [ "100", "hello" ] + + - match: + hits.hits.1._source: + name: "B" + my_values: + k: ["55", "66"] + long: [77, 88] + - match: + hits.hits.1.fields: + c.copy: ["55", "66", "77", "88"] + + - match: + hits.hits.2._source: + name: "C" + my_values: + k: "hello" + long: 100 + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy: [ "100", "hello", "zap" ] + +--- +synthetic_source with copy_to pointing to ambiguous field: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing to ambiguous field and subobjects false: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + subobjects: false + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing to ambiguous field and subobjects auto: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + subobjects: auto + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing at dynamic field: + - requires: + test_runner_features: contains + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + k: + type: keyword + copy_to: c.copy + c: + properties: + f: + type: float + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + k: ["55", "66"] + + - do: + index: + index: test + id: 3 + refresh: true + body: + k: "hello" + c: + copy: "zap" + + - do: + search: + index: test + body: + docvalue_fields: [ "c.copy.keyword" ] + + - match: + hits.hits.0._source: + k: "hello" + - match: + hits.hits.0.fields: + c.copy.keyword: [ "hello" ] + + - match: + hits.hits.1._source: + k: ["55", "66"] + - match: + hits.hits.1.fields: + c.copy.keyword: [ "55", "66" ] + + - match: + hits.hits.2._source: + k: "hello" + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy.keyword: [ "hello", "zap" ] + +--- +synthetic_source with copy_to pointing inside dynamic object: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + k: + type: keyword + copy_to: c.copy + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + k: ["55", "66"] + + - do: + index: + index: test + id: 3 + refresh: true + body: + k: "hello" + c: + copy: "zap" + + - do: + search: + index: test + body: + docvalue_fields: [ "c.copy.keyword" ] + + - match: + hits.hits.0._source: + k: "hello" + - match: + hits.hits.0.fields: + c.copy.keyword: [ "hello" ] + + - match: + hits.hits.1._source: + k: ["55", "66"] + - match: + hits.hits.1.fields: + c.copy.keyword: [ "55", "66" ] + + - match: + hits.hits.2._source: + k: "hello" + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy.keyword: [ "hello", "zap" ] + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml new file mode 100644 index 0000000000000..e2c3006232c53 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml @@ -0,0 +1,121 @@ +--- +flattened ignore_above single-value field: + - requires: + cluster_features: [ "flattened.ignore_above_support" ] + reason: introduce ignore_above support in flattened fields + - do: + indices.create: + index: test + body: + mappings: + properties: + keyword: + type: keyword + ignore_above: 5 + flat: + type: flattened + ignore_above: 5 + + - do: + index: + index: test + id: "1" + refresh: true + body: + keyword: "foo" + flat: { "value": "foo", "key": "foo key" } + + - do: + index: + index: test + id: "2" + refresh: true + body: + keyword: "foo bar" + flat: { "value": "foo bar", "key": "foo bar key"} + + - do: + search: + index: test + body: + fields: + - keyword + - flat + query: + match_all: {} + + - match: { hits.total.value: 2 } + + - match: { hits.hits.0._source.keyword: "foo" } + - match: { hits.hits.0._source.flat.value: "foo" } + - match: { hits.hits.0._source.flat.key: "foo key" } + - match: { hits.hits.1._source.keyword: "foo bar" } + - match: { hits.hits.1._source.flat.value: "foo bar" } + - match: { hits.hits.1._source.flat.key: "foo bar key" } + + - match: { hits.hits.0.fields.keyword.0: "foo" } + - match: { hits.hits.0.fields.flat.0.value: "foo" } + - match: { hits.hits.1.fields.keyword.0: null } + - match: { hits.hits.1.fields.flat.0.value: null } + +--- +flattened ignore_above multi-value field: + - requires: + cluster_features: [ "flattened.ignore_above_support" ] + reason: introduce ignore_above support in flattened fields + - do: + indices.create: + index: test + body: + mappings: + properties: + keyword: + type: keyword + ignore_above: 5 + flat: + type: flattened + ignore_above: 5 + + - do: + index: + index: test + id: "1" + refresh: true + body: + keyword: ["foo","bar"] + flat: { "value": ["foo", "bar"], "key": "foo bar array key" } + + - do: + index: + index: test + id: "2" + refresh: true + body: + keyword: ["foobar", "foo", "bar"] + flat: { "value": ["foobar", "foo"], "key": ["foo key", "bar key"]} + + - do: + search: + index: test + body: + fields: + - keyword + - flat + query: + match_all: { } + + - match: { hits.total.value: 2 } + + - match: { hits.hits.0._source.keyword: ["foo", "bar"] } + - match: { hits.hits.0._source.flat.value: ["foo", "bar"] } + - match: { hits.hits.0._source.flat.key: "foo bar array key" } + - match: { hits.hits.1._source.keyword: ["foobar", "foo", "bar"] } + - match: { hits.hits.1._source.flat.value: ["foobar", "foo"] } + - match: { hits.hits.1._source.flat.key: ["foo key", "bar key"] } + + - match: { hits.hits.0.fields.keyword: [ "foo", "bar" ] } + - match: { hits.hits.0.fields.flat.0.value: ["foo", "bar"] } + - match: { hits.hits.0.fields.flat.0.key: null } + - match: { hits.hits.1.fields.keyword: [ "foo", "bar" ] } + - match: { hits.hits.1.fields.flat.0.value: "foo" } + - match: { hits.hits.1.fields.flat.0.key: null } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 6eb7a8dcad7aa..616afd3cf67ad 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -119,11 +119,11 @@ missing dimension on routing path field: type: keyword --- -multi-value routing path field: +multi-value routing path field succeeds: - requires: test_runner_features: close_to - cluster_features: ["gte_v8.13.0"] - reason: _tsid hashing introduced in 8.13 + cluster_features: ["routing.multi_value_routing_path"] + reason: support for multi-value dimensions - do: indices.create: @@ -172,12 +172,7 @@ multi-value routing path field: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "unmapped_field": 40, "tag": [ "one", "three" ] }' - - is_true: errors - - - match: {items.1.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } - - match: {items.3.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } - - match: {items.4.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } - - match: {items.7.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" } + - is_false: errors - do: search: @@ -195,13 +190,21 @@ multi-value routing path field: avg: field: voltage - - match: {hits.total.value: 4} - - length: {aggregations.tsids.buckets: 2} + - match: {hits.total.value: 8} + - length: {aggregations.tsids.buckets: 4} - - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } + - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUtt0uPSOYoRw_LI4DD7DFEGEJ3NR3eQkMY" } - match: {aggregations.tsids.buckets.0.doc_count: 2 } - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } + - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUtt0uPSddqA4WYKglGPR_C0cJe8QGaiC2c" } - match: {aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }} + + - match: { aggregations.tsids.buckets.2.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } + - match: {aggregations.tsids.buckets.2.doc_count: 2 } + - close_to: {aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }} + + - match: { aggregations.tsids.buckets.3.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } + - match: {aggregations.tsids.buckets.3.doc_count: 2 } + - close_to: {aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index 0ad254937d0dc..91674b7ce9050 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -15,12 +15,14 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; @@ -57,7 +59,7 @@ public void testMappingValidationIndexExists() { } """; indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -87,13 +89,125 @@ public void testMappingValidationIndexExists() { assertThat(fields.size(), equalTo(1)); } + @SuppressWarnings("unchecked") + public void testMappingValidationIndexExistsWithComponentTemplate() throws IOException { + /* + * This test simulates a BulkRequest of two documents into an existing index. Then we make sure the index contains no documents, and + * that the index's mapping in the cluster state has not been updated with the two new field. With the mapping from the template + * that was used to create the index, we would expect the second document to throw an exception because it uses a field that does + * not exist. But we substitute a new version of the component template named "test-component-template" that allows for the new + * field. + */ + String originalComponentTemplateMappingString = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + CompressedXContent mapping = CompressedXContent.fromJSON(originalComponentTemplateMappingString); + Template template = new Template(Settings.EMPTY, mapping, null); + PutComponentTemplateAction.Request componentTemplateActionRequest = new PutComponentTemplateAction.Request( + "test-component-template" + ); + ComponentTemplate componentTemplate = new ComponentTemplate(template, null, null); + componentTemplateActionRequest.componentTemplate(componentTemplate); + client().execute(PutComponentTemplateAction.INSTANCE, componentTemplateActionRequest).actionGet(); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-index-*")) + .componentTemplates(List.of("test-component-template")) + .build(); + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + request.indexTemplate(composableIndexTemplate); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + + String indexName = "my-index-1"; + // First, run before the index is created: + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + // Now, create the index and make sure the component template substitutions work the same: + indicesAdmin().create(new CreateIndexRequest(indexName)).actionGet(); + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + // Now make sure nothing was actually changed: + indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); + SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet(); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); + searchResponse.decRef(); + ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet(); + Map indexMapping = clusterStateResponse.getState().metadata().index(indexName).mapping().sourceAsMap(); + Map fields = (Map) indexMapping.get("properties"); + assertThat(fields.size(), equalTo(1)); + } + + private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName) { + IndexRequest indexRequest1 = new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID()); + IndexRequest indexRequest2 = new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID()); + { + // First we use the original component template, and expect a failure in the second document: + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + + { + // Now we substitute a "test-component-template" that defines both fields, so we expect no exception: + BulkRequest bulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of( + "test-component-template", + Map.of( + "template", + Map.of( + "mappings", + Map.of( + "dynamic", + "strict", + "properties", + Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text")) + ) + ) + ) + ) + ); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + } + public void testMappingValidationIndexDoesNotExistsNoTemplate() { /* * This test simulates a BulkRequest of two documents into an index that does not exist. There is no template (other than the * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. */ String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -140,7 +254,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -172,7 +286,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() { indicesAdmin().putTemplate( new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") ).actionGet(); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -226,7 +340,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); { // First, try with no @timestamp to make sure we're picking up data-stream-specific templates - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -252,7 +366,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti } { // Now with @timestamp - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "@timestamp": "2024-08-27", diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 132dfeab46d93..f1079e0cc4974 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -220,6 +220,7 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_IBM_WATSONX_EMBEDDINGS_ADDED = def(8_744_00_0); public static final TransportVersion BULK_INCREMENTAL_STATE = def(8_745_00_0); public static final TransportVersion FAILURE_STORE_STATUS_IN_INDEX_RESPONSE = def(8_746_00_0); + public static final TransportVersion ESQL_AGGREGATION_OPERATOR_STATUS_FINISH_NANOS = def(8_747_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java index e98091beb88a5..c6d60f34eddeb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java @@ -9,32 +9,25 @@ package org.elasticsearch.action.admin.indices.alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse.AliasActionResult; -import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.metadata.AliasAction; +import org.elasticsearch.core.TimeValue; import java.util.List; +import java.util.Objects; /** * Cluster state update request that allows to add or remove aliases */ -public class IndicesAliasesClusterStateUpdateRequest extends ClusterStateUpdateRequest { - private final List actions; - - private final List actionResults; - - public IndicesAliasesClusterStateUpdateRequest(List actions, List actionResults) { - this.actions = actions; - this.actionResults = actionResults; - } - - /** - * Returns the alias actions to be performed - */ - public List actions() { - return actions; - } - - public List getActionResults() { - return actionResults; +public record IndicesAliasesClusterStateUpdateRequest( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + List actions, + List actionResults +) { + public IndicesAliasesClusterStateUpdateRequest { + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + Objects.requireNonNull(actions); + Objects.requireNonNull(actionResults); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 129e853ec638c..c73c44e9a23e7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -255,9 +255,11 @@ protected void masterOperation( } request.aliasActions().clear(); IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest( + request.masterNodeTimeout(), + request.ackTimeout(), unmodifiableList(finalActions), unmodifiableList(actionResults) - ).ackTimeout(request.ackTimeout()).masterNodeTimeout(request.masterNodeTimeout()); + ); indexAliasesService.indicesAliases(updateRequest, listener.delegateResponse((l, e) -> { logger.debug("failed to perform aliases", e); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index fdced5fc18ac9..3561a4d0e2cb4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; @@ -156,7 +157,8 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + Map.of() ); final Map> overlapping = new HashMap<>(); @@ -233,7 +235,8 @@ public static Template resolveTemplate( final NamedXContentRegistry xContentRegistry, final IndicesService indicesService, final SystemIndices systemIndices, - Set indexSettingProviders + Set indexSettingProviders, + Map componentTemplateSubstitutions ) throws Exception { var metadata = simulatedState.getMetadata(); Settings templateSettings = resolveSettings(simulatedState.metadata(), matchingTemplate); @@ -263,6 +266,7 @@ public static Template resolveTemplate( null, // empty request mapping as the user can't specify any explicit mappings via the simulate api simulatedState, matchingTemplate, + componentTemplateSubstitutions, xContentRegistry, simulatedIndexName ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 30bbad0b57df0..af7a253b5a042 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -170,7 +170,8 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + Map.of() ); if (request.includeDefaults()) { listener.onResponse( diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index 78652081c9f0d..5eae1c660d7d0 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -39,6 +40,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; +import java.util.Map; import java.util.Objects; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; @@ -168,19 +171,21 @@ public void onTimeout(TimeValue timeout) { private void forkAndExecute(Task task, BulkRequest bulkRequest, Executor executor, ActionListener releasingListener) { executor.execute(new ActionRunnable<>(releasingListener) { @Override - protected void doRun() { + protected void doRun() throws IOException { applyPipelinesAndDoInternalExecute(task, bulkRequest, executor, releasingListener); } }); } - private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) { + private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) + throws IOException { boolean hasIndexRequestsWithPipelines = false; final Metadata metadata = clusterService.state().getMetadata(); + Map templateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); for (DocWriteRequest actionRequest : bulkRequest.requests) { IndexRequest indexRequest = getIndexWriteRequest(actionRequest); if (indexRequest != null) { - IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata); + IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata, templateSubstitutions); hasIndexRequestsWithPipelines |= IngestService.hasPipeline(indexRequest); } @@ -250,7 +255,7 @@ private void processBulkIndexIngestRequest( } else { ActionRunnable runnable = new ActionRunnable<>(actionListener) { @Override - protected void doRun() { + protected void doRun() throws IOException { applyPipelinesAndDoInternalExecute(task, bulkRequest, executor, actionListener); } @@ -328,7 +333,7 @@ private void applyPipelinesAndDoInternalExecute( BulkRequest bulkRequest, Executor executor, ActionListener listener - ) { + ) throws IOException { final long relativeStartTimeNanos = relativeTimeNanos(); if (applyPipelines(task, bulkRequest, executor, listener) == false) { doInternalExecute(task, bulkRequest, executor, listener, relativeStartTimeNanos); @@ -349,6 +354,6 @@ protected abstract void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ); + ) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 3199dd7f76b2d..03768af029141 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -53,6 +53,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -208,7 +209,11 @@ protected void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ) { + ) throws IOException { + assert (bulkRequest instanceof SimulateBulkRequest) == false + : "TransportBulkAction should never be called with a SimulateBulkRequest"; + assert bulkRequest.getComponentTemplateSubstitutions().isEmpty() + : "Component template substitutions are not allowed in a non-simulated bulk"; trackIndexRequests(bulkRequest); Map indicesToAutoCreate = new HashMap<>(); Set dataStreamsToBeRolledOver = new HashSet<>(); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index ada47f9de098c..0ea763c215959 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -49,6 +50,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; @@ -108,13 +110,16 @@ protected void doInternalExecute( Executor executor, ActionListener listener, long relativeStartTimeNanos - ) { + ) throws IOException { + assert bulkRequest instanceof SimulateBulkRequest + : "TransportSimulateBulkAction should only ever be called with a SimulateBulkRequest but got a " + bulkRequest.getClass(); final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); + Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; IndexRequest request = (IndexRequest) docRequest; - Exception mappingValidationException = validateMappings(request); + Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, request); responses.set( i, BulkItemResponse.success( @@ -140,10 +145,11 @@ protected void doInternalExecute( /** * This creates a temporary index with the mappings of the index in the request, and then attempts to index the source from the request * into it. If there is a mapping exception, that exception is returned. On success the returned exception is null. + * @parem componentTemplateSubstitutions The component template definitions to use in place of existing ones for validation * @param request The IndexRequest whose source will be validated against the mapping (if it exists) of its index * @return a mapping exception if the source does not match the mappings, otherwise null */ - private Exception validateMappings(IndexRequest request) { + private Exception validateMappings(Map componentTemplateSubstitutions, IndexRequest request) { final SourceToParse sourceToParse = new SourceToParse( request.id(), request.source(), @@ -157,7 +163,11 @@ private Exception validateMappings(IndexRequest request) { Exception mappingValidationException = null; IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); try { - if (indexAbstraction != null) { + if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty()) { + /* + * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService + * to do the mapping validation, using all the existing logic for validation. + */ IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); indicesService.withTempIndexService(imd, indexService -> { indexService.mapperService().updateMapping(null, imd); @@ -178,23 +188,29 @@ private Exception validateMappings(IndexRequest request) { }); } else { /* - * The index did not exist, so we put together the mappings from existing templates. - * This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). However, - * it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the path for - * when the index does not exist). And it does not deal with system indices since we do not intend for users to simulate - * writing to system indices. + * The index did not exist, or we have component template substitutions, so we put together the mappings from existing + * templates This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). + * However, it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the + * path for when the index does not exist). And it does not deal with system indices since we do not intend for users to + * simulate writing to system indices. */ + // First, we remove the index from the cluster state if necessary (since we're going to use the templates) + ClusterState simulatedState = indexAbstraction == null + ? state + : new ClusterState.Builder(state).metadata(Metadata.builder(state.metadata()).remove(request.index()).build()).build(); + String matchingTemplate = findV2Template(state.metadata(), request.index(), false); if (matchingTemplate != null) { final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( matchingTemplate, request.index(), - state, + simulatedState, isDataStreamsLifecycleOnlyMode(clusterService.getSettings()), xContentRegistry, indicesService, systemIndices, - indexSettingProviders + indexSettingProviders, + componentTemplateSubstitutions ); CompressedXContent mappings = template.mappings(); if (mappings != null) { diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java index a69c4470c2add..11872cd85d9ba 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.support.master; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.TimeValue; @@ -23,9 +22,7 @@ * Abstract base class for action requests that track acknowledgements of cluster state updates: such a request is acknowledged only once * the cluster state update is committed and all relevant nodes have applied it and acknowledged its application to the elected master.. */ -public abstract class AcknowledgedRequest> extends MasterNodeRequest - implements - AckedRequest { +public abstract class AcknowledgedRequest> extends MasterNodeRequest { public static final TimeValue DEFAULT_ACK_TIMEOUT = timeValueSeconds(30); @@ -74,7 +71,6 @@ public final Request ackTimeout(TimeValue ackTimeout) { /** * @return the current ack timeout as a {@link TimeValue} */ - @Override public final TimeValue ackTimeout() { return ackTimeout; } diff --git a/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java b/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java index 34d7b6e913dec..de42591c15d27 100644 --- a/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java @@ -9,8 +9,8 @@ package org.elasticsearch.cluster; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Priority; import org.elasticsearch.core.TimeValue; @@ -23,21 +23,38 @@ public abstract class AckedClusterStateUpdateTask extends ClusterStateUpdateTask implements ClusterStateAckListener { private final ActionListener listener; - private final AckedRequest request; + private final TimeValue ackTimeout; - protected AckedClusterStateUpdateTask(AckedRequest request, ActionListener listener) { - this(Priority.NORMAL, request, listener); + protected AckedClusterStateUpdateTask(AcknowledgedRequest request, ActionListener listener) { + this(Priority.NORMAL, request.masterNodeTimeout(), request.ackTimeout(), listener); + } + + protected AckedClusterStateUpdateTask( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActionListener listener + ) { + this(Priority.NORMAL, masterNodeTimeout, ackTimeout, listener); + } + + protected AckedClusterStateUpdateTask( + Priority priority, + AcknowledgedRequest request, + ActionListener listener + ) { + this(priority, request.masterNodeTimeout(), request.ackTimeout(), listener); } @SuppressWarnings("unchecked") protected AckedClusterStateUpdateTask( Priority priority, - AckedRequest request, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, ActionListener listener ) { - super(priority, request.masterNodeTimeout()); + super(priority, masterNodeTimeout); this.listener = (ActionListener) listener; - this.request = request; + this.ackTimeout = ackTimeout; } /** @@ -81,6 +98,6 @@ public void onFailure(Exception e) { * Acknowledgement timeout, maximum time interval to wait for acknowledgements */ public final TimeValue ackTimeout() { - return request.ackTimeout(); + return ackTimeout; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java b/server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java deleted file mode 100644 index 1a9618dd59f4d..0000000000000 --- a/server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.cluster.ack; - -import org.elasticsearch.core.TimeValue; - -/** - * Identifies a cluster state update request with acknowledgement support - */ -public interface AckedRequest { - - /** - * Returns the acknowledgement timeout - */ - TimeValue ackTimeout(); - - /** - * Returns the timeout for the request to be completed on the master node - */ - TimeValue masterNodeTimeout(); -} diff --git a/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java index 69a51c80839d0..8841b315b0138 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java @@ -15,7 +15,7 @@ * Base class to be used when needing to update the cluster state * Contains the basic fields that are always needed */ -public abstract class ClusterStateUpdateRequest> implements AckedRequest { +public abstract class ClusterStateUpdateRequest> { private TimeValue ackTimeout; private TimeValue masterNodeTimeout; @@ -23,7 +23,6 @@ public abstract class ClusterStateUpdateRequest(listener, threadPool.getThreadContext()); submitUnbatchedTask( "create-data-stream [" + request.name + "]", - new AckedClusterStateUpdateTask(Priority.HIGH, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.HIGH, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { // When we're manually creating a data stream (i.e. not an auto creation), we don't need to initialize the failure store @@ -138,13 +142,19 @@ public ClusterState createDataStream( ); } - public static final class CreateDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest< - CreateDataStreamClusterStateUpdateRequest> { - - private final boolean performReroute; - private final String name; - private final long startTime; - private final SystemDataStreamDescriptor descriptor; + public record CreateDataStreamClusterStateUpdateRequest( + String name, + long startTime, + @Nullable SystemDataStreamDescriptor systemDataStreamDescriptor, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + boolean performReroute + ) { + public CreateDataStreamClusterStateUpdateRequest { + Objects.requireNonNull(name); + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + } public CreateDataStreamClusterStateUpdateRequest(String name) { this(name, System.currentTimeMillis(), null, TimeValue.ZERO, TimeValue.ZERO, true); @@ -154,42 +164,14 @@ public CreateDataStreamClusterStateUpdateRequest( String name, SystemDataStreamDescriptor systemDataStreamDescriptor, TimeValue masterNodeTimeout, - TimeValue timeout, + TimeValue ackTimeout, boolean performReroute ) { - this(name, System.currentTimeMillis(), systemDataStreamDescriptor, masterNodeTimeout, timeout, performReroute); - } - - public CreateDataStreamClusterStateUpdateRequest( - String name, - long startTime, - SystemDataStreamDescriptor systemDataStreamDescriptor, - TimeValue masterNodeTimeout, - TimeValue timeout, - boolean performReroute - ) { - this.name = name; - this.startTime = startTime; - this.descriptor = systemDataStreamDescriptor; - this.performReroute = performReroute; - masterNodeTimeout(masterNodeTimeout); - ackTimeout(timeout); + this(name, System.currentTimeMillis(), systemDataStreamDescriptor, masterNodeTimeout, ackTimeout, performReroute); } public boolean isSystem() { - return descriptor != null; - } - - public boolean performReroute() { - return performReroute; - } - - public SystemDataStreamDescriptor getSystemDataStreamDescriptor() { - return descriptor; - } - - long getStartTime() { - return startTime; + return systemDataStreamDescriptor != null; } } @@ -238,7 +220,7 @@ static ClusterState createDataStream( boolean initializeFailureStore ) throws Exception { String dataStreamName = request.name; - SystemDataStreamDescriptor systemDataStreamDescriptor = request.getSystemDataStreamDescriptor(); + SystemDataStreamDescriptor systemDataStreamDescriptor = request.systemDataStreamDescriptor(); boolean isSystemDataStreamName = metadataCreateIndexService.getSystemIndices().isSystemDataStream(request.name); assert (isSystemDataStreamName && systemDataStreamDescriptor != null) || (isSystemDataStreamName == false && systemDataStreamDescriptor == null) @@ -287,13 +269,13 @@ static ClusterState createDataStream( if (isSystem) { throw new IllegalArgumentException("Failure stores are not supported on system data streams"); } - String failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, initialGeneration, request.getStartTime()); + String failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, initialGeneration, request.startTime()); currentState = createFailureStoreIndex( metadataCreateIndexService, "initialize_data_stream", settings, currentState, - request.getStartTime(), + request.startTime(), dataStreamName, template, failureStoreIndexName, @@ -303,7 +285,7 @@ static ClusterState createDataStream( } if (writeIndex == null) { - String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, initialGeneration, request.getStartTime()); + String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, initialGeneration, request.startTime()); currentState = createBackingIndex( metadataCreateIndexService, currentState, @@ -392,7 +374,7 @@ private static ClusterState createBackingIndex( firstBackingIndexName ).dataStreamName(dataStreamName) .systemDataStreamDescriptor(systemDataStreamDescriptor) - .nameResolvedInstant(request.getStartTime()) + .nameResolvedInstant(request.startTime()) .performReroute(request.performReroute()) .setMatchingTemplate(template); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 07cfcddac9c93..061aa18dd464a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -296,7 +296,12 @@ private void onlyCreateIndex(final CreateIndexClusterStateUpdateRequest request, var delegate = new AllocationActionListener<>(listener, threadPool.getThreadContext()); submitUnbatchedTask( "create-index [" + request.index() + "], cause [" + request.cause() + "]", - new AckedClusterStateUpdateTask(Priority.URGENT, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.URGENT, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -648,6 +653,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( request.mappings(), currentState, templateName, + Map.of(), xContentRegistry, request.index() ); @@ -806,6 +812,7 @@ private static List collectSystemV2Mappings( List templateMappings = MetadataIndexTemplateService.collectMappings( composableIndexTemplate, componentTemplates, + Map.of(), indexName ); return collectV2Mappings(null, templateMappings, xContentRegistry); @@ -815,10 +822,16 @@ public static List collectV2Mappings( @Nullable final String requestMappings, final ClusterState currentState, final String templateName, + Map componentTemplateSubstitutions, final NamedXContentRegistry xContentRegistry, final String indexName ) throws Exception { - List templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName); + List templateMappings = MetadataIndexTemplateService.collectMappings( + currentState, + templateName, + componentTemplateSubstitutions, + indexName + ); return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 130a38f39b11a..f926e2b6ebf35 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -282,7 +282,7 @@ public boolean mustAck(DiscoveryNode discoveryNode) { @Override public void onAllNodesAcked() { - listener.onResponse(IndicesAliasesResponse.build(request.getActionResults())); + listener.onResponse(IndicesAliasesResponse.build(request.actionResults())); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 207f773b54f34..9888059af9686 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -698,7 +698,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT final var now = Instant.now(); final var metadata = currentState.getMetadata(); - final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), "tmp_idx"); + final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), Map.of(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, metadata.componentTemplates()); // First apply settings sourced from index setting providers: for (var provider : indexSettingProviders) { @@ -1348,7 +1348,12 @@ private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, Compo /** * Collect the given v2 template into an ordered list of mappings. */ - public static List collectMappings(final ClusterState state, final String templateName, final String indexName) { + public static List collectMappings( + final ClusterState state, + final String templateName, + Map componentTemplateSubstitutions, + final String indexName + ) { final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName); assert template != null : "attempted to resolve mappings for a template [" + templateName + "] that did not exist in the cluster state"; @@ -1357,7 +1362,7 @@ public static List collectMappings(final ClusterState state, } final Map componentTemplates = state.metadata().componentTemplates(); - return collectMappings(template, componentTemplates, indexName); + return collectMappings(template, componentTemplates, componentTemplateSubstitutions, indexName); } /** @@ -1366,6 +1371,7 @@ public static List collectMappings(final ClusterState state, public static List collectMappings( final ComposableIndexTemplate template, final Map componentTemplates, + final Map componentTemplateSubstitutions, final String indexName ) { Objects.requireNonNull(template, "Composable index template must be provided"); @@ -1376,9 +1382,12 @@ public static List collectMappings( ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_MAPPING_SNIPPET ); } + final Map combinedComponentTemplates = new HashMap<>(); + combinedComponentTemplates.putAll(componentTemplates); + combinedComponentTemplates.putAll(componentTemplateSubstitutions); List mappings = template.composedOf() .stream() - .map(componentTemplates::get) + .map(combinedComponentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::mappings) @@ -1428,24 +1437,44 @@ public static Settings resolveSettings(final List templat * Resolve the given v2 template into a collected {@link Settings} object */ public static Settings resolveSettings(final Metadata metadata, final String templateName) { + return resolveSettings(metadata, templateName, Map.of()); + } + + public static Settings resolveSettings( + final Metadata metadata, + final String templateName, + Map templateSubstitutions + ) { final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); assert template != null : "attempted to resolve settings for a template [" + templateName + "] that did not exist in the cluster state"; if (template == null) { return Settings.EMPTY; } - return resolveSettings(template, metadata.componentTemplates()); + return resolveSettings(template, metadata.componentTemplates(), templateSubstitutions); } /** * Resolve the provided v2 template and component templates into a collected {@link Settings} object */ public static Settings resolveSettings(ComposableIndexTemplate template, Map componentTemplates) { + return resolveSettings(template, componentTemplates, Map.of()); + } + + public static Settings resolveSettings( + ComposableIndexTemplate template, + Map componentTemplates, + Map templateSubstitutions + ) { Objects.requireNonNull(template, "attempted to resolve settings for a null template"); Objects.requireNonNull(componentTemplates, "attempted to resolve settings with null component templates"); + Map combinedComponentTemplates = new HashMap<>(); + combinedComponentTemplates.putAll(componentTemplates); + // We want any substitutions to take precedence: + combinedComponentTemplates.putAll(templateSubstitutions); List componentSettings = template.composedOf() .stream() - .map(componentTemplates::get) + .map(combinedComponentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::settings) @@ -1694,7 +1723,7 @@ private static void validateCompositeTemplate( String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; // Parse mappings to ensure they are valid after being composed - List mappings = collectMappings(stateWithIndex, templateName, indexName); + List mappings = collectMappings(stateWithIndex, templateName, Map.of(), indexName); try { MapperService mapperService = tempIndexService.mapperService(); mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 1c93a13583e94..aee60c3eda57f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -18,7 +18,6 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; import org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener; import org.elasticsearch.cluster.service.ClusterService; @@ -40,6 +39,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -105,7 +105,12 @@ public void migrateToDataStream( var delegate = new AllocationActionListener<>(listener, threadContext); submitUnbatchedTask( "migrate-to-data-stream [" + request.aliasName + "]", - new AckedClusterStateUpdateTask(Priority.HIGH, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.HIGH, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { @@ -278,15 +283,11 @@ static void validateBackingIndices(ClusterState currentState, String dataStreamN } } - @SuppressWarnings("rawtypes") - public static final class MigrateToDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest { - - private final String aliasName; - - public MigrateToDataStreamClusterStateUpdateRequest(String aliasName, TimeValue masterNodeTimeout, TimeValue timeout) { - this.aliasName = aliasName; - masterNodeTimeout(masterNodeTimeout); - ackTimeout(timeout); + public record MigrateToDataStreamClusterStateUpdateRequest(String aliasName, TimeValue masterNodeTimeout, TimeValue ackTimeout) { + public MigrateToDataStreamClusterStateUpdateRequest { + Objects.requireNonNull(aliasName); + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index f4ed7ec6c264c..cb55e65abbd18 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -35,7 +35,6 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -45,6 +44,7 @@ import java.util.function.Predicate; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.common.xcontent.XContentParserUtils.expectValueToken; /** * Generates the shard id for {@code (id, routing)} pairs. @@ -52,6 +52,7 @@ public abstract class IndexRouting { static final NodeFeature BOOLEAN_ROUTING_PATH = new NodeFeature("routing.boolean_routing_path"); + static final NodeFeature MULTI_VALUE_ROUTING_PATH = new NodeFeature("routing.multi_value_routing_path"); /** * Build the routing from {@link IndexMetadata}. @@ -301,7 +302,13 @@ public String createId(Map flat, byte[] suffix) { Builder b = builder(); for (Map.Entry e : flat.entrySet()) { if (isRoutingPath.test(e.getKey())) { - b.hashes.add(new NameAndHash(new BytesRef(e.getKey()), hash(new BytesRef(e.getValue().toString())))); + if (e.getValue() instanceof List listValue) { + for (Object v : listValue) { + b.addHash(e.getKey(), new BytesRef(v.toString())); + } + } else { + b.addHash(e.getKey(), new BytesRef(e.getValue().toString())); + } } } return b.createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); @@ -336,7 +343,7 @@ public class Builder { public void addMatching(String fieldName, BytesRef string) { if (isRoutingPath.test(fieldName)) { - hashes.add(new NameAndHash(new BytesRef(fieldName), hash(string))); + addHash(fieldName, string); } } @@ -357,6 +364,13 @@ private void extractObject(@Nullable String path, XContentParser source) throws } } + private void extractArray(@Nullable String path, XContentParser source) throws IOException { + while (source.currentToken() != Token.END_ARRAY) { + expectValueToken(source.currentToken(), source); + extractItem(path, source); + } + } + private void extractItem(String path, XContentParser source) throws IOException { switch (source.currentToken()) { case START_OBJECT: @@ -367,7 +381,12 @@ private void extractItem(String path, XContentParser source) throws IOException case VALUE_STRING: case VALUE_NUMBER: case VALUE_BOOLEAN: - hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text())))); + addHash(path, new BytesRef(source.text())); + source.nextToken(); + break; + case START_ARRAY: + source.nextToken(); + extractArray(path, source); source.nextToken(); break; case VALUE_NULL: @@ -376,28 +395,24 @@ private void extractItem(String path, XContentParser source) throws IOException default: throw new ParsingException( source.getTokenLocation(), - "Routing values must be strings but found [{}]", + "Cannot extract routing path due to unexpected token [{}]", source.currentToken() ); } } + private void addHash(String path, BytesRef value) { + hashes.add(new NameAndHash(new BytesRef(path), hash(value), hashes.size())); + } + private int buildHash(IntSupplier onEmpty) { - Collections.sort(hashes); - Iterator itr = hashes.iterator(); - if (itr.hasNext() == false) { + if (hashes.isEmpty()) { return onEmpty.getAsInt(); } - NameAndHash prev = itr.next(); - int hash = hash(prev.name) ^ prev.hash; - while (itr.hasNext()) { - NameAndHash next = itr.next(); - if (prev.name.equals(next.name)) { - throw new IllegalArgumentException("Duplicate routing dimension for [" + next.name + "]"); - } - int thisHash = hash(next.name) ^ next.hash; - hash = 31 * hash + thisHash; - prev = next; + Collections.sort(hashes); + int hash = 0; + for (NameAndHash nah : hashes) { + hash = 31 * hash + (hash(nah.name) ^ nah.hash); } return hash; } @@ -458,10 +473,13 @@ private String error(String operation) { } } - private record NameAndHash(BytesRef name, int hash) implements Comparable { + private record NameAndHash(BytesRef name, int hash, int order) implements Comparable { @Override public int compareTo(NameAndHash o) { - return name.compareTo(o.name); + int i = name.compareTo(o.name); + if (i != 0) return i; + // ensures array values are in the order as they appear in the source + return Integer.compare(order, o.order); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java index 8885410bd0530..f8028ce7f9d68 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java @@ -18,6 +18,6 @@ public class RoutingFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(IndexRouting.BOOLEAN_ROUTING_PATH); + return Set.of(IndexRouting.BOOLEAN_ROUTING_PATH, IndexRouting.MULTI_VALUE_ROUTING_PATH); } } diff --git a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java index 09a72b0810364..2bf4a07d85133 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java @@ -388,10 +388,16 @@ public static ZonedDateTime nowWithMillisResolution(Clock clock) { return ZonedDateTime.now(millisResolutionClock); } + private static final boolean USES_COMPAT = System.getProperty("java.locale.providers", "").contains("COMPAT"); // check for all textual fields, and localized zone offset // the weird thing with Z is to ONLY match 4 in a row, with no Z before or after (but those groups can also be empty) - private static final Predicate CONTAINS_CHANGING_TEXT_SPECIFIERS = System.getProperty("java.locale.providers", "") - .contains("COMPAT") ? Pattern.compile("[BEGOavz]|LLL|MMM|QQQ|qqq|ccc|eee|(? CONTAINS_CHANGING_TEXT_SPECIFIERS = USES_COMPAT + ? Pattern.compile("[BEGOavz]|LLL|MMM|QQQ|qqq|ccc|eee|(? CONTAINS_WEEK_DATE_SPECIFIERS = USES_COMPAT + ? Pattern.compile("[YWw]").asPredicate() + : Predicates.never(); @UpdateForV9 // this can be removed, we will only use CLDR on v9 static void checkTextualDateFormats(String format) { @@ -403,5 +409,13 @@ static void checkTextualDateFormats(String format) { format ); } + if (CONTAINS_WEEK_DATE_SPECIFIERS.test(format)) { + deprecationLogger.warn( + DeprecationCategory.PARSING, + "cldr_week_dates_" + format, + "Date format [{}] contains week-date field specifiers that are changing in JDK 23", + format + ); + } } } diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java index cd7cf5ddbd893..6390e62f9758f 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java @@ -72,6 +72,20 @@ public static void ensureExpectedToken(Token expected, Token actual, XContentPar } } + /** + * Makes sure the provided token {@linkplain Token#isValue() is a value type} + * + * @throws ParsingException if the token is not a value type + */ + public static void expectValueToken(Token actual, XContentParser parser) { + if (actual.isValue() == false) { + throw new ParsingException( + parser.getTokenLocation(), + String.format(Locale.ROOT, "Failed to parse object: expecting value token but found [%s]", actual) + ); + } + } + private static ParsingException parsingException(XContentParser parser, Token expected, Token actual) { return new ParsingException( parser.getTokenLocation(), diff --git a/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java b/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java index 77454c5686538..ccf375aec60a5 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java +++ b/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java @@ -33,6 +33,24 @@ public interface HttpPreRequest { */ String uri(); + /** + * The uri without the query string. + */ + default String rawPath() { + String uri = uri(); + final int index = uri.indexOf('?'); + if (index >= 0) { + return uri.substring(0, index); + } else { + final int index2 = uri.indexOf('#'); + if (index2 >= 0) { + return uri.substring(0, index2); + } else { + return uri; + } + } + } + /** * Get all of the headers and values associated with the HTTP headers. * Modifications of this map are not supported. diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java index 7e5494007b369..71d9768ac5ff7 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java @@ -533,7 +533,7 @@ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOExcepti int i, docValueCount; @Override - public long nextValue() throws IOException { + public long nextValue() { return ords[i++]; } @@ -543,7 +543,7 @@ public int docValueCount() { } @Override - public boolean advanceExact(int target) throws IOException { + public boolean advanceExact(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java index 597fabc5418f2..d46f8a3f20a4f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java @@ -10,13 +10,9 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.search.DocIdSetIterator; - -import java.io.IOException; /** - * Base implementation that throws an {@link IOException} for the - * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * Base implementation. This impl is safe to use for sorting and * aggregations, which only use {@link #advanceExact(int)} and * {@link #binaryValue()}. */ @@ -28,12 +24,12 @@ public int docID() { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java index e6925dde709b8..fca1adc137b71 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java @@ -23,12 +23,12 @@ public abstract class AbstractNumericDocValues extends NumericDocValues { @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java index e5e75cae041e4..05bb9a8fe74c4 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java @@ -23,12 +23,12 @@ public abstract class AbstractSortedDocValues extends SortedDocValues { @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java index a1c3a054fdd27..4c17715ec55f8 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java @@ -33,7 +33,7 @@ public int nextDoc() throws IOException { } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java index 90d577c21568e..c040a2e8dc2d0 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java @@ -28,12 +28,12 @@ public int docID() { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java index 2684b40f4e442..076b2fecf4bef 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java @@ -36,12 +36,12 @@ public int docID() { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException(); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/BinaryScriptFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/BinaryScriptFieldData.java index b618268d9a5b0..1a612bbe724f9 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/BinaryScriptFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/BinaryScriptFieldData.java @@ -66,9 +66,5 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() { - - } } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/DoubleScriptFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/DoubleScriptFieldData.java index 3b8cc9956442a..742e0a0f95078 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/DoubleScriptFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/DoubleScriptFieldData.java @@ -43,7 +43,7 @@ public DoubleScriptFieldData build(IndexFieldDataCache cache, CircuitBreakerServ } private final String fieldName; - DoubleFieldScript.LeafFactory leafFactory; + final DoubleFieldScript.LeafFactory leafFactory; private final ToScriptFieldFactory toScriptFieldFactory; private DoubleScriptFieldData( diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index b090354b08459..6c8d52ac18280 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -42,12 +42,12 @@ public static SortedBinaryDocValues emptySortedBinary() { public static NumericDoubleValues emptyNumericDouble() { return new NumericDoubleValues() { @Override - public boolean advanceExact(int doc) throws IOException { + public boolean advanceExact(int doc) { return false; } @Override - public double doubleValue() throws IOException { + public double doubleValue() { throw new UnsupportedOperationException(); } @@ -243,14 +243,6 @@ public static BinaryDocValues unwrapSingleton(SortedBinaryDocValues values) { return null; } - /** - * Returns whether the provided values *might* be multi-valued. There is no - * guarantee that this method will return {@code false} in the single-valued case. - */ - public static boolean isMultiValued(SortedSetDocValues values) { - return DocValues.unwrapSingleton(values) == null; - } - /** * Return a {@link String} representation of the provided values. That is * typically used for scripts or for the `map` execution mode of terms aggs. @@ -634,7 +626,7 @@ public boolean advanceExact(int target) throws IOException { } @Override - public long longValue() throws IOException { + public long longValue() { return value; } }; @@ -661,7 +653,7 @@ public boolean advanceExact(int target) throws IOException { } @Override - public double doubleValue() throws IOException { + public double doubleValue() { return value; } }; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptDocValues.java index 25c28a99a600c..c9aed46377c3f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptDocValues.java @@ -67,6 +67,6 @@ public int docValueCount() { public long nextValue() { int lat = GeoEncodingUtils.encodeLatitude(script.lats()[cursor]); int lon = GeoEncodingUtils.encodeLongitude(script.lons()[cursor++]); - return Long.valueOf((((long) lat) << 32) | (lon & 0xFFFFFFFFL)); + return (((long) lat) << 32) | (lon & 0xFFFFFFFFL); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptFieldData.java index 1ffb0deca8a3d..e6bd6c3e59656 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/GeoPointScriptFieldData.java @@ -103,10 +103,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() { - - } }; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 902eba0f7d5d8..65eb724549bfc 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -20,7 +20,6 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.Closeable; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -90,7 +89,6 @@ public synchronized void clearField(final String fieldName) { * Returns fielddata for the provided field type, given the provided fully qualified index name, while also making * a {@link SearchLookup} supplier available that is required for runtime fields. */ - @SuppressWarnings("unchecked") public > IFD getForField(MappedFieldType fieldType, FieldDataContext fieldDataContext) { return getFromBuilder(fieldType, fieldType.fielddataBuilder(fieldDataContext)); } @@ -134,7 +132,7 @@ public void setListener(IndexFieldDataCache.Listener listener) { } @Override - public void close() throws IOException { + public void close() { clear(); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index e1e8dad62daed..289f1dd6abd25 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -198,20 +198,16 @@ private XFieldComparatorSource comparatorSource( MultiValueMode sortMode, Nested nested ) { - switch (targetNumericType) { - case HALF_FLOAT: - case FLOAT: - return new FloatValuesComparatorSource(this, missingValue, sortMode, nested); - case DOUBLE: - return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested); - case DATE: - return dateComparatorSource(missingValue, sortMode, nested); - case DATE_NANOSECONDS: - return dateNanosComparatorSource(missingValue, sortMode, nested); - default: + return switch (targetNumericType) { + case HALF_FLOAT, FLOAT -> new FloatValuesComparatorSource(this, missingValue, sortMode, nested); + case DOUBLE -> new DoubleValuesComparatorSource(this, missingValue, sortMode, nested); + case DATE -> dateComparatorSource(missingValue, sortMode, nested); + case DATE_NANOSECONDS -> dateNanosComparatorSource(missingValue, sortMode, nested); + default -> { assert targetNumericType.isFloatingPoint() == false; - return new LongValuesComparatorSource(this, missingValue, sortMode, nested, targetNumericType); - } + yield new LongValuesComparatorSource(this, missingValue, sortMode, nested, targetNumericType); + } + }; } protected XFieldComparatorSource dateComparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java index 2c48eb28e7fe5..e42db267920ed 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java @@ -32,6 +32,9 @@ public interface LeafFieldData extends Accountable, Releasable { */ SortedBinaryDocValues getBytesValues(); + @Override + default void close() {} + /** * Return a formatted representation of the values */ diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/LongScriptFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LongScriptFieldData.java index 2ad11aa771a5c..7f05ca0e50f92 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/LongScriptFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/LongScriptFieldData.java @@ -20,8 +20,6 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import java.io.IOException; - public final class LongScriptFieldData extends IndexNumericFieldData { public static class Builder implements IndexFieldData.Builder { @@ -79,7 +77,7 @@ public LongScriptLeafFieldData load(LeafReaderContext context) { } @Override - public LongScriptLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + public LongScriptLeafFieldData loadDirect(LeafReaderContext context) { return new LongScriptLeafFieldData(new LongScriptDocValues(leafFactory.newInstance(context)), toScriptFieldFactory); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java index 449b3b9ced777..06deda44dc364 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java @@ -45,7 +45,6 @@ public GeoPoint nextValue() throws IOException { /** * Returns a single-valued view of the {@link MultiPointValues} if possible, otherwise null. */ - @Override protected GeoPointValues getPointValues() { final NumericDocValues singleton = DocValues.unwrapSingleton(numericValues); return singleton != null ? new GeoPointValues(singleton) : null; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiPointValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiPointValues.java index f99683ebe0831..854f06d9b606a 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiPointValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiPointValues.java @@ -53,8 +53,4 @@ public int docValueCount() { */ public abstract T nextValue() throws IOException; - /** - * Returns a single-valued view of the {@link MultiPointValues} if possible, otherwise null. - */ - protected abstract PointValues getPointValues(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java b/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java index 15817ae513146..14a1f287eb500 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/RamAccountingTermsEnum.java @@ -52,7 +52,7 @@ public RamAccountingTermsEnum( * Always accept the term. */ @Override - protected AcceptStatus accept(BytesRef term) throws IOException { + protected AcceptStatus accept(BytesRef term) { return AcceptStatus.YES; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java index d5933c7fab4d2..02f5de7e3c8ca 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java @@ -25,7 +25,7 @@ public abstract class SortingNumericDocValues extends SortedNumericDocValues { protected long[] values; protected int valuesCursor; private final Sorter sorter; - private LongConsumer circuitBreakerConsumer; + private final LongConsumer circuitBreakerConsumer; protected SortingNumericDocValues() { this(l -> {}); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java index d9f6cefeba38c..bb4eb3f0bebce 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java @@ -132,11 +132,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() { - - } - @Override public SortedBinaryDocValues getBytesValues() { throw new IllegalArgumentException("not supported for source fallback"); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java index a4ff63ae1e04e..bfb5c58cac3af 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java @@ -61,7 +61,7 @@ protected SourceValueFetcherMultiGeoPointIndexFieldData( } @Override - public SourceValueFetcherMultiGeoPointLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public SourceValueFetcherMultiGeoPointLeafFieldData loadDirect(LeafReaderContext context) { return new SourceValueFetcherMultiGeoPointLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java index 3f26a6c71197d..e24a6c5251f8f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java @@ -62,7 +62,7 @@ protected SourceValueFetcherSortedBinaryIndexFieldData( } @Override - public SourceValueFetcherSortedBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public SourceValueFetcherSortedBinaryLeafFieldData loadDirect(LeafReaderContext context) { return new SourceValueFetcherSortedBinaryLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider); } @@ -127,7 +127,7 @@ public int docValueCount() { } @Override - public BytesRef nextValue() throws IOException { + public BytesRef nextValue() { assert iterator.hasNext(); return iterator.next(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java index c28890cb82671..56937d47b794d 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java @@ -59,7 +59,7 @@ protected SourceValueFetcherSortedBooleanIndexFieldData( } @Override - public SourceValueFetcherLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public SourceValueFetcherLeafFieldData loadDirect(LeafReaderContext context) { return new SourceValueFetcherSortedBooleanLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider); } @@ -129,7 +129,7 @@ public int docValueCount() { } @Override - public long nextValue() throws IOException { + public long nextValue() { assert iteratorIndex < trueCount + falseCount; return iteratorIndex++ < falseCount ? 0L : 1L; } @@ -140,12 +140,12 @@ public int docID() { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException("not supported for source fallback"); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException("not supported for source fallback"); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java index 2df4917390354..f4bc2216217a7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java @@ -61,7 +61,7 @@ protected SourceValueFetcherSortedDoubleIndexFieldData( } @Override - public SourceValueFetcherLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public SourceValueFetcherLeafFieldData loadDirect(LeafReaderContext context) { return new SourceValueFetcherSortedDoubleLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider); } @@ -128,7 +128,7 @@ public int docValueCount() { } @Override - public double nextValue() throws IOException { + public double nextValue() { assert iterator.hasNext(); return iterator.next(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java index cf4cbf93b8e8d..ce1dff33e80ce 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java @@ -62,7 +62,7 @@ protected SourceValueFetcherSortedNumericIndexFieldData( } @Override - public SourceValueFetcherSortedNumericLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public SourceValueFetcherSortedNumericLeafFieldData loadDirect(LeafReaderContext context) { return new SourceValueFetcherSortedNumericLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider); } @@ -129,7 +129,7 @@ public int docValueCount() { } @Override - public long nextValue() throws IOException { + public long nextValue() { assert iterator.hasNext(); return iterator.next(); } @@ -140,12 +140,12 @@ public int docID() { } @Override - public int nextDoc() throws IOException { + public int nextDoc() { throw new UnsupportedOperationException("not supported for source fallback"); } @Override - public int advance(int target) throws IOException { + public int advance(int target) { throw new UnsupportedOperationException("not supported for source fallback"); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java index 6d3d5b988c9f9..a0c4efe9ef6c1 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldIndexFieldData.java @@ -104,9 +104,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() {} - @Override public SortedBinaryDocValues getBytesValues() { throw new IllegalArgumentException("not supported for source fallback"); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java index 69e348956822e..550833c396430 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StoredFieldSortedBinaryIndexFieldData.java @@ -75,7 +75,7 @@ public int docValueCount() { } @Override - public BytesRef nextValue() throws IOException { + public BytesRef nextValue() { assert current < docValueCount; return sorted.get(current++); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index 459075cfcc5b1..f1e4ba95e76fe 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -54,7 +54,7 @@ public SortField.Type reducedType() { return SortField.Type.FLOAT; } - private NumericDoubleValues getNumericDocValues(LeafReaderContext context, float missingValue) throws IOException { + private NumericDoubleValues getNumericDocValues(LeafReaderContext context, double missingValue) throws IOException { final SortedNumericDoubleValues values = indexFieldData.load(context).getDoubleValues(); if (nested == null) { return FieldData.replaceMissing(sortMode.select(values), missingValue); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java index e857a715ee657..def1c40e1e6f0 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java @@ -111,8 +111,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() {} }; subs[i] = atomicFD[i].getOrdinalsValues(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java index 2ffc8527b179a..5b3e523a10f41 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java @@ -76,7 +76,7 @@ public IndexOrdinalsFieldData newConsumer(DirectoryReader source) { } @Override - public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) { throw new IllegalStateException("loadDirect(LeafReaderContext) should not be called in this context"); } @@ -86,7 +86,7 @@ public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) { } @Override - public IndexOrdinalsFieldData loadGlobalDirect(DirectoryReader indexReader) throws Exception { + public IndexOrdinalsFieldData loadGlobalDirect(DirectoryReader indexReader) { return this; } @@ -179,7 +179,7 @@ private TermsEnum[] getOrLoadTermsEnums() { } @Override - public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafOrdinalsFieldData loadDirect(LeafReaderContext context) { return load(context); } @@ -189,7 +189,7 @@ public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) { } @Override - public IndexOrdinalsFieldData loadGlobalDirect(DirectoryReader indexReader) throws Exception { + public IndexOrdinalsFieldData loadGlobalDirect(DirectoryReader indexReader) { return this; } @@ -258,8 +258,6 @@ public Collection getChildResources() { return segmentAfd[context.ord].getChildResources(); } - @Override - public void close() {} }; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java index d39e811ae49f4..e1b66ec379d8e 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java @@ -16,7 +16,6 @@ import org.apache.lucene.util.packed.PagedGrowableWriter; import java.io.Closeable; -import java.io.IOException; import java.util.Arrays; /** @@ -369,7 +368,7 @@ public int maxDoc() { * Closes this builder and release all resources. */ @Override - public void close() throws IOException { + public void close() { ordinals = null; } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java index d609796d58c83..63fa0e21c6cb6 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java @@ -68,8 +68,4 @@ public BytesRef nextValue() throws IOException { }; } - @Override - public void close() { - // no-op - } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java index 5227a4bed1f1c..5038117a5d554 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java @@ -83,13 +83,7 @@ public LeafOrdinalsFieldData load(LeafReaderContext context) { try { return cache.load(context, this); } catch (Exception e) { - if (e instanceof ElasticsearchException) { - throw (ElasticsearchException) e; - } else if (e instanceof ExecutionException && e.getCause() instanceof ElasticsearchException) { - throw (ElasticsearchException) e.getCause(); - } else { - throw new ElasticsearchException(e); - } + throw handleCacheLoadException(e); } } @@ -131,13 +125,7 @@ private IndexOrdinalsFieldData loadGlobalInternal(DirectoryReader indexReader) { try { return cache.load(indexReader, this); } catch (Exception e) { - if (e instanceof ElasticsearchException) { - throw (ElasticsearchException) e; - } else if (e instanceof ExecutionException && e.getCause() instanceof ElasticsearchException) { - throw (ElasticsearchException) e.getCause(); - } else { - throw new ElasticsearchException(e); - } + throw handleCacheLoadException(e); } } @@ -157,6 +145,16 @@ public boolean supportsGlobalOrdinalsMapping() { return false; } + private static ElasticsearchException handleCacheLoadException(Exception e) { + if (e instanceof ElasticsearchException ese) { + return ese; + } + if (e instanceof ExecutionException && e.getCause() instanceof ElasticsearchException ese) { + throw ese; + } + throw new ElasticsearchException(e); + } + /** * A {@code PerValueEstimator} is a sub-class that can be used to estimate * the memory overhead for loading the data. Each field data diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java index 8dc30f2646a8a..d1280040f085f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java @@ -43,9 +43,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() {} - @Override public SortedSetDocValues getOrdinalsValues() { return DocValues.emptySortedSet(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java index 22bceca37fb6c..54f1ecac89f8e 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java @@ -47,11 +47,6 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { return new DelegateDocValuesField(new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(getBytesValues())), name); } - @Override - public void close() { - // no-op - } - @Override public long ramBytesUsed() { return 0; // unknown diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryIndexFieldData.java index f414f800bf809..0d90ef9a41071 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryIndexFieldData.java @@ -65,7 +65,7 @@ public BinaryDVLeafFieldData load(LeafReaderContext context) { } @Override - public BinaryDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public BinaryDVLeafFieldData loadDirect(LeafReaderContext context) { return load(context); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryIndexFieldData.java index 2f0f7254f1265..f5d4bf837f062 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryIndexFieldData.java @@ -75,7 +75,7 @@ public BytesBinaryDVLeafFieldData load(LeafReaderContext context) { } @Override - public BytesBinaryDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public BytesBinaryDVLeafFieldData loadDirect(LeafReaderContext context) { return load(context); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java index 85adc2b5c5012..8e15e6fc8e7f1 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/ConstantIndexFieldData.java @@ -113,9 +113,6 @@ public int docID() { return DocValues.singleton(sortedValues); } - @Override - public void close() {} - } private final ConstantLeafFieldData atomicFieldData; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/FormattedSortedNumericDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/FormattedSortedNumericDocValues.java new file mode 100644 index 0000000000000..da64520c7c86c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/FormattedSortedNumericDocValues.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.search.DocValueFormat; + +import java.io.IOException; + +public final class FormattedSortedNumericDocValues implements FormattedDocValues { + private final SortedNumericDocValues values; + private final DocValueFormat format; + + public FormattedSortedNumericDocValues(SortedNumericDocValues values, DocValueFormat format) { + this.values = values; + this.format = format; + } + + @Override + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); + } + + @Override + public int docValueCount() { + return values.docValueCount(); + } + + @Override + public Object nextValue() throws IOException { + return format.format(values.nextValue()); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java index f303c61c86b6e..64e6dd223caf4 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVLeafFieldData.java @@ -31,11 +31,6 @@ public long ramBytesUsed() { return 0; // not exposed by lucene } - @Override - public void close() { - // noop - } - @Override public SortedNumericDocValues getSortedNumericDocValues() { try { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointIndexFieldData.java index 40f934746126a..cbf7d76f24137 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointIndexFieldData.java @@ -43,7 +43,7 @@ public LeafPointFieldData load(LeafReaderContext context) { } @Override - public LeafPointFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafPointFieldData loadDirect(LeafReaderContext context) { return load(context); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java index 144dc9220110c..e870f7af22562 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java @@ -60,7 +60,4 @@ public Object nextValue() throws IOException { }; } - @Override - public void close() {} - } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java index 5c849fa1efaa6..bad40d2c51943 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.fielddata.plain; -import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FormattedDocValues; import org.elasticsearch.index.fielddata.LeafNumericFieldData; @@ -17,8 +16,6 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.DocValueFormat; -import java.io.IOException; - /** * Specialization of {@link LeafNumericFieldData} for integers. */ @@ -47,25 +44,7 @@ public final SortedNumericDoubleValues getDoubleValues() { @Override public FormattedDocValues getFormattedValues(DocValueFormat format) { - SortedNumericDocValues values = getLongValues(); - return new FormattedDocValues() { - @Override - public boolean advanceExact(int docId) throws IOException { - return values.advanceExact(docId); - } - - @Override - public int docValueCount() throws IOException { - return values.docValueCount(); - } - - @Override - public Object nextValue() throws IOException { - return format.format(values.nextValue()); - } - }; + return new FormattedSortedNumericDocValues(getLongValues(), format); } - @Override - public void close() {} } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java index 1dc2203e04c7d..d0b77c70eb0cf 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesLeafFieldData.java @@ -38,9 +38,6 @@ public PagedBytesLeafFieldData( this.ordinals = ordinals; } - @Override - public void close() {} - @Override public long ramBytesUsed() { long size = ordinals.ramBytesUsed(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedDoublesIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedDoublesIndexFieldData.java index 151ecb7d376e3..ae20100f4c1f7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedDoublesIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedDoublesIndexFieldData.java @@ -112,7 +112,7 @@ public NumericType getNumericType() { } @Override - public LeafNumericFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafNumericFieldData loadDirect(LeafReaderContext context) { return load(context); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java index d72ca60bb9786..98fd4e27f9422 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -153,7 +153,7 @@ public NumericType getNumericType() { } @Override - public LeafNumericFieldData loadDirect(LeafReaderContext context) throws Exception { + public LeafNumericFieldData loadDirect(LeafReaderContext context) { return load(context); } @@ -206,25 +206,9 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { @Override public FormattedDocValues getFormattedValues(DocValueFormat format) { - DocValueFormat nanosFormat = DocValueFormat.withNanosecondResolution(format); - SortedNumericDocValues values = getLongValuesAsNanos(); - return new FormattedDocValues() { - @Override - public boolean advanceExact(int docId) throws IOException { - return values.advanceExact(docId); - } - - @Override - public int docValueCount() throws IOException { - return values.docValueCount(); - } - - @Override - public Object nextValue() throws IOException { - return nanosFormat.format(values.nextValue()); - } - }; + return new FormattedSortedNumericDocValues(getLongValuesAsNanos(), DocValueFormat.withNanosecondResolution(format)); } + } /** @@ -264,4 +248,5 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { return toScriptFieldFactory.getScriptFieldFactory(getLongValues(), name); } } + } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetBytesLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetBytesLeafFieldData.java index 9b08c319ecd98..d4682e680a54a 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetBytesLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetBytesLeafFieldData.java @@ -40,9 +40,6 @@ public SortedSetDocValues getOrdinalsValues() { } } - @Override - public void close() {} - @Override public long ramBytesUsed() { return 0; // unknown diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java index 954f2db34f4ba..7c8b058cb01f1 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java @@ -82,7 +82,7 @@ public BucketedSort newBucketedSort( } @Override - public StringBinaryDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + public StringBinaryDVLeafFieldData loadDirect(LeafReaderContext context) { return load(context); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 4d1b68214eddb..c2970d8716147 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -123,6 +123,7 @@ public int get() { private Field version; private final SeqNoFieldMapper.SequenceIDFields seqID; private final Set fieldsAppliedFromTemplates; + /** * Fields that are copied from values of other fields via copy_to. * This per-document state is needed since it is possible @@ -453,26 +454,16 @@ public boolean isFieldAppliedFromTemplate(String name) { public void markFieldAsCopyTo(String fieldName) { copyToFields.add(fieldName); - if (mappingLookup.isSourceSynthetic() && indexSettings().getSkipIgnoredSourceWrite() == false) { - /* - Mark this field as containing copied data meaning it should not be present - in synthetic _source (to be consistent with stored _source). - Ignored source values take precedence over standard synthetic source implementation - so by adding this nothing entry we "disable" field in synthetic source. - Otherwise, it would be constructed f.e. from doc_values which leads to duplicate values - in copied field after reindexing. - - Note that this applies to fields that are copied from fields using ignored source themselves - and therefore we don't check for canAddIgnoredField(). - */ - ignoredFieldValues.add(IgnoredSourceFieldMapper.NameValue.fromContext(this, fieldName, XContentDataHelper.nothing())); - } } public boolean isCopyToDestinationField(String name) { return copyToFields.contains(name); } + public Set getCopyToFields() { + return copyToFields; + } + /** * Add a new mapper dynamically created while parsing. * diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 7b926b091b3a2..d57edb757ba10 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -26,9 +26,8 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; -import java.util.Comparator; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Stream; @@ -113,6 +112,10 @@ NameValue cloneWithValue(BytesRef value) { assert value() == null; return new NameValue(name, parentOffset, value, doc); } + + boolean hasValue() { + return XContentDataHelper.isDataPresent(value); + } } static final class IgnoredValuesFieldMapperType extends StringFieldType { @@ -147,11 +150,38 @@ protected String contentType() { @Override public void postParse(DocumentParserContext context) { // Ignored values are only expected in synthetic mode. - assert context.getIgnoredFieldValues().isEmpty() || context.mappingLookup().isSourceSynthetic(); - List ignoredFieldValues = new ArrayList<>(context.getIgnoredFieldValues()); - // ensure consistent ordering when retrieving synthetic source - Collections.sort(ignoredFieldValues, Comparator.comparing(NameValue::name)); - for (NameValue nameValue : ignoredFieldValues) { + if (context.mappingLookup().isSourceSynthetic() == false) { + assert context.getIgnoredFieldValues().isEmpty(); + return; + } + + Collection ignoredValuesToWrite = context.getIgnoredFieldValues(); + if (context.getCopyToFields().isEmpty() == false && indexSettings.getSkipIgnoredSourceWrite() == false) { + /* + Mark fields as containing copied data meaning they should not be present + in synthetic _source (to be consistent with stored _source). + Ignored source values take precedence over standard synthetic source implementation + so by adding the `XContentDataHelper.voidValue()` entry we disable the field in synthetic source. + Otherwise, it would be constructed f.e. from doc_values which leads to duplicate values + in copied field after reindexing. + */ + var mutableList = new ArrayList<>(ignoredValuesToWrite); + for (String copyToField : context.getCopyToFields()) { + ObjectMapper parent = context.parent().findParentMapper(copyToField); + if (parent == null) { + // There are scenarios when this can happen: + // 1. all values of the field that is the source of copy_to are null + // 2. copy_to points at a field inside a disabled object + // 3. copy_to points at dynamic field which is not yet applied to mapping, we will process it properly on re-parse. + continue; + } + int offset = parent.isRoot() ? 0 : parent.fullPath().length() + 1; + mutableList.add(new IgnoredSourceFieldMapper.NameValue(copyToField, offset, XContentDataHelper.voidValue(), context.doc())); + } + ignoredValuesToWrite = mutableList; + } + + for (NameValue nameValue : ignoredValuesToWrite) { nameValue.doc().add(new StoredField(NAME, encode(nameValue))); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index cf3265502a4db..d18c3283ef909 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -11,6 +11,7 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.util.Set; @@ -38,7 +39,9 @@ public Set getFeatures() { SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX, Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE, SourceFieldMapper.SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT, - SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_FIX + SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_FIX, + FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT, + SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index e4ce38d6cec0b..f9c854749e885 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -808,6 +808,42 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep } + ObjectMapper findParentMapper(String leafFieldPath) { + var pathComponents = leafFieldPath.split("\\."); + int startPathComponent = 0; + + ObjectMapper current = this; + String pathInCurrent = leafFieldPath; + + while (current != null) { + if (current.mappers.containsKey(pathInCurrent)) { + return current; + } + + // Go one level down if possible + var parent = current; + current = null; + + var childMapperName = new StringBuilder(); + for (int i = startPathComponent; i < pathComponents.length - 1; i++) { + if (childMapperName.isEmpty() == false) { + childMapperName.append("."); + } + childMapperName.append(pathComponents[i]); + + var childMapper = parent.mappers.get(childMapperName.toString()); + if (childMapper instanceof ObjectMapper objectMapper) { + current = objectMapper; + startPathComponent = i + 1; + pathInCurrent = pathInCurrent.substring(childMapperName.length() + 1); + break; + } + } + } + + return null; + } + protected SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream mappers, boolean isFragment) { var fields = mappers.sorted(Comparator.comparing(Mapper::fullPath)) .map(Mapper::syntheticFieldLoader) @@ -828,10 +864,18 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader { private final List fields; private final boolean isFragment; + private boolean storedFieldLoadersHaveValues; private boolean docValuesLoadersHaveValues; private boolean ignoredValuesPresent; private List ignoredValues; + // If this loader has anything to write. + // In special cases this can be false even if doc values loaders or stored field loaders + // have values. + // F.e. objects that only contain fields that are destinations of copy_to. + private boolean writersHaveValues; + // Use an ordered map between field names and writers to order writing by field name. + private TreeMap currentWriters; private SyntheticSourceFieldLoader(List fields, boolean isFragment) { this.fields = fields; @@ -882,9 +926,55 @@ public boolean advanceToDoc(int docId) throws IOException { } } + @Override + public void prepare() { + if ((storedFieldLoadersHaveValues || docValuesLoadersHaveValues || ignoredValuesPresent) == false) { + writersHaveValues = false; + return; + } + + for (var loader : fields) { + // Currently this logic is only relevant for object loaders. + if (loader instanceof ObjectMapper.SyntheticSourceFieldLoader objectSyntheticFieldLoader) { + objectSyntheticFieldLoader.prepare(); + } + } + + currentWriters = new TreeMap<>(); + + if (ignoredValues != null && ignoredValues.isEmpty() == false) { + for (IgnoredSourceFieldMapper.NameValue value : ignoredValues) { + if (value.hasValue()) { + writersHaveValues |= true; + } + + var existing = currentWriters.get(value.name()); + if (existing == null) { + currentWriters.put(value.name(), new FieldWriter.IgnoredSource(value)); + } else if (existing instanceof FieldWriter.IgnoredSource isw) { + isw.mergeWith(value); + } + } + } + + for (SourceLoader.SyntheticFieldLoader field : fields) { + if (field.hasValue()) { + if (currentWriters.containsKey(field.fieldName()) == false) { + writersHaveValues |= true; + currentWriters.put(field.fieldName(), new FieldWriter.FieldLoader(field)); + } else { + // Skip if the field source is stored separately, to avoid double-printing. + // Make sure to reset the state of loader so that values stored inside will not + // be used after this document is finished. + field.reset(); + } + } + } + } + @Override public boolean hasValue() { - return storedFieldLoadersHaveValues || docValuesLoadersHaveValues || ignoredValuesPresent; + return writersHaveValues; } @Override @@ -892,12 +982,13 @@ public void write(XContentBuilder b) throws IOException { if (hasValue() == false) { return; } + if (isRoot() && isEnabled() == false) { // If the root object mapper is disabled, it is expected to contain // the source encapsulated within a single ignored source value. assert ignoredValues.size() == 1 : ignoredValues.size(); XContentDataHelper.decodeAndWrite(b, ignoredValues.get(0).value()); - ignoredValues = null; + softReset(); return; } @@ -907,41 +998,12 @@ public void write(XContentBuilder b) throws IOException { b.startObject(leafName()); } - if (ignoredValues != null && ignoredValues.isEmpty() == false) { - // Use an ordered map between field names and writer functions, to order writing by field name. - Map orderedFields = new TreeMap<>(); - for (IgnoredSourceFieldMapper.NameValue value : ignoredValues) { - var existing = orderedFields.get(value.name()); - if (existing == null) { - orderedFields.put(value.name(), new FieldWriter.IgnoredSource(value)); - } else if (existing instanceof FieldWriter.IgnoredSource isw) { - isw.mergeWith(value); - } - } - for (SourceLoader.SyntheticFieldLoader field : fields) { - if (field.hasValue()) { - if (orderedFields.containsKey(field.fieldName()) == false) { - orderedFields.put(field.fieldName(), new FieldWriter.FieldLoader(field)); - } else { - // Skip if the field source is stored separately, to avoid double-printing. - // Make sure to reset the state of loader so that values stored inside will not - // be used after this document is finished. - field.reset(); - } - } - } - - for (var writer : orderedFields.values()) { + for (var writer : currentWriters.values()) { + if (writer.hasValue()) { writer.writeTo(b); } - ignoredValues = null; - } else { - for (SourceLoader.SyntheticFieldLoader field : fields) { - if (field.hasValue()) { - field.write(b); - } - } } + b.endObject(); softReset(); } @@ -957,6 +1019,8 @@ private void softReset() { storedFieldLoadersHaveValues = false; docValuesLoadersHaveValues = false; ignoredValuesPresent = false; + ignoredValues = null; + writersHaveValues = false; } @Override @@ -986,34 +1050,49 @@ public String fieldName() { interface FieldWriter { void writeTo(XContentBuilder builder) throws IOException; + boolean hasValue(); + record FieldLoader(SourceLoader.SyntheticFieldLoader loader) implements FieldWriter { @Override public void writeTo(XContentBuilder builder) throws IOException { loader.write(builder); } + + @Override + public boolean hasValue() { + return loader.hasValue(); + } } class IgnoredSource implements FieldWriter { private final String fieldName; private final String leafName; - private final List values; + private final List encodedValues; IgnoredSource(IgnoredSourceFieldMapper.NameValue initialValue) { this.fieldName = initialValue.name(); this.leafName = initialValue.getFieldName(); - this.values = new ArrayList<>(); - this.values.add(initialValue.value()); + this.encodedValues = new ArrayList<>(); + if (initialValue.hasValue()) { + this.encodedValues.add(initialValue.value()); + } } @Override public void writeTo(XContentBuilder builder) throws IOException { - XContentDataHelper.writeMerged(builder, leafName, values); + XContentDataHelper.writeMerged(builder, leafName, encodedValues); + } + + @Override + public boolean hasValue() { + return encodedValues.isEmpty() == false; } public FieldWriter mergeWith(IgnoredSourceFieldMapper.NameValue nameValue) { assert Objects.equals(nameValue.name(), fieldName) : "IgnoredSource is merged with wrong field data"; - - values.add(nameValue.value()); + if (nameValue.hasValue()) { + encodedValues.add(nameValue.value()); + } return this; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 3318595ed7129..118cdbffc5db9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -48,6 +48,9 @@ public class SourceFieldMapper extends MetadataFieldMapper { "mapper.source.synthetic_source_with_copy_to_and_doc_values_false" ); public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_FIX = new NodeFeature("mapper.source.synthetic_source_copy_to_fix"); + public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX = new NodeFeature( + "mapper.source.synthetic_source_copy_to_inside_objects_fix" + ); public static final String NAME = "_source"; public static final String RECOVERY_SOURCE_NAME = "_recovery_source"; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index baff3835d104b..ec255a53e7c5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -209,6 +209,9 @@ public void write(LeafStoredFieldLoader storedFieldLoader, int docId, XContentBu if (docValuesLoader != null) { docValuesLoader.advanceToDoc(docId); } + + loader.prepare(); + // TODO accept a requested xcontent type if (loader.hasValue()) { loader.write(b); @@ -299,6 +302,16 @@ public String fieldName() { */ DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException; + /** + Perform any preprocessing needed before producing synthetic source + and deduce whether this mapper (and its children, if any) have values to write. + The expectation is for this method to be called before {@link SyntheticFieldLoader#hasValue()} + and {@link SyntheticFieldLoader#write(XContentBuilder)} are used. + */ + default void prepare() { + // Noop + } + /** * Has this field loaded any values for this document? */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 80535be39a4c4..d660b5b952645 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -41,13 +41,14 @@ import java.io.IOException; import java.net.InetAddress; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Base64; import java.util.Collections; -import java.util.Comparator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; +import java.util.SortedMap; +import java.util.TreeMap; /** * Mapper for {@code _tsid} field included generated when the index is @@ -176,16 +177,14 @@ public static class TimeSeriesIdBuilder implements DocumentDimensions { public static final int MAX_DIMENSIONS = 512; - private record Dimension(BytesRef name, BytesReference value) {} - private final Murmur3Hasher tsidHasher = new Murmur3Hasher(0); /** - * A sorted set of the serialized values of dimension fields that will be used + * A map of the serialized values of dimension fields that will be used * for generating the _tsid field. The map will be used by {@link TimeSeriesIdFieldMapper} * to build the _tsid field for the document. */ - private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.name)); + private final SortedMap> dimensions = new TreeMap<>(); /** * Builds the routing. Used for building {@code _id}. If null then skipped. */ @@ -203,9 +202,17 @@ public BytesReference buildLegacyTsid() throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(dimensions.size()); - for (Dimension entry : dimensions) { - out.writeBytesRef(entry.name); - entry.value.writeTo(out); + for (Map.Entry> entry : dimensions.entrySet()) { + out.writeBytesRef(entry.getKey()); + List value = entry.getValue(); + if (value.size() > 1) { + // multi-value dimensions are only supported for newer indices that use buildTsidHash + throw new IllegalArgumentException( + "Dimension field [" + entry.getKey().utf8ToString() + "] cannot be a multi-valued field." + ); + } + assert value.isEmpty() == false : "dimension value is empty"; + value.get(0).writeTo(out); } return out.bytes(); } @@ -237,18 +244,19 @@ public BytesReference buildTsidHash() { int tsidHashIndex = StreamOutput.putVInt(tsidHash, len, 0); tsidHasher.reset(); - for (final Dimension dimension : dimensions) { - tsidHasher.update(dimension.name.bytes); + for (final BytesRef name : dimensions.keySet()) { + tsidHasher.update(name.bytes); } tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); // NOTE: concatenate all dimension value hashes up to a certain number of dimensions int tsidHashStartIndex = tsidHashIndex; - for (final Dimension dimension : dimensions) { + for (final List values : dimensions.values()) { if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) { break; } - final BytesRef dimensionValueBytesRef = dimension.value.toBytesRef(); + assert values.isEmpty() == false : "dimension values are empty"; + final BytesRef dimensionValueBytesRef = values.get(0).toBytesRef(); ByteUtils.writeIntLE( StringHelper.murmurhash3_x86_32( dimensionValueBytesRef.bytes, @@ -264,8 +272,10 @@ public BytesReference buildTsidHash() { // NOTE: hash all dimension field allValues tsidHasher.reset(); - for (final Dimension dimension : dimensions) { - tsidHasher.update(dimension.value.toBytesRef().bytes); + for (final List values : dimensions.values()) { + for (BytesReference v : values) { + tsidHasher.update(v.toBytesRef().bytes); + } } tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); @@ -368,8 +378,20 @@ public DocumentDimensions validate(final IndexSettings settings) { } private void add(String fieldName, BytesReference encoded) throws IOException { - if (dimensions.add(new Dimension(new BytesRef(fieldName), encoded)) == false) { - throw new IllegalArgumentException("Dimension field [" + fieldName + "] cannot be a multi-valued field."); + BytesRef name = new BytesRef(fieldName); + List values = dimensions.get(name); + if (values == null) { + // optimize for the common case where dimensions are not multi-valued + dimensions.put(name, List.of(encoded)); + } else { + if (values.size() == 1) { + // converts the immutable list that's optimized for the common case of having only one value to a mutable list + BytesReference previousValue = values.get(0); + values = new ArrayList<>(4); + values.add(previousValue); + dimensions.put(name, values); + } + values.add(encoded); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java index 354f0ec92b0fa..8bacaf8505f91 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java @@ -72,7 +72,7 @@ public static BytesRef encodeXContentBuilder(XContentBuilder builder) throws IOE } /** - * Returns a special encoded value that signals that values of this field + * Returns a special encoded value that signals that this field * should not be present in synthetic source. * * An example is a field that has values copied to it using copy_to. @@ -80,7 +80,7 @@ public static BytesRef encodeXContentBuilder(XContentBuilder builder) throws IOE * synthetic _source same as it wouldn't be present in stored source. * @return */ - public static BytesRef nothing() { + public static BytesRef voidValue() { return new BytesRef(new byte[] { VOID_ENCODING }); } @@ -112,41 +112,27 @@ static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { /** * Writes encoded values to provided builder. If there are multiple values they are merged into * a single resulting array. + * + * Note that this method assumes all encoded parts have values that need to be written (are not VOID encoded). * @param b destination * @param fieldName name of the field that is written * @param encodedParts subset of field data encoded using methods of this class. Can contain arrays which will be flattened. * @throws IOException */ static void writeMerged(XContentBuilder b, String fieldName, List encodedParts) throws IOException { - var partsWithData = 0; - for (BytesRef encodedPart : encodedParts) { - if (isDataPresent(encodedPart)) { - partsWithData++; - } - } - - if (partsWithData == 0) { + if (encodedParts.isEmpty()) { return; } - if (partsWithData == 1) { + if (encodedParts.size() == 1) { b.field(fieldName); - for (BytesRef encodedPart : encodedParts) { - if (isDataPresent(encodedPart)) { - XContentDataHelper.decodeAndWrite(b, encodedPart); - } - } - + XContentDataHelper.decodeAndWrite(b, encodedParts.get(0)); return; } b.startArray(fieldName); for (var encodedValue : encodedParts) { - if (isDataPresent(encodedValue) == false) { - continue; - } - Optional encodedXContentType = switch ((char) encodedValue.bytes[encodedValue.offset]) { case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> Optional.of( getXContentType(encodedValue) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 2b13d78e9c9b2..867a4a7ec39e0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -73,9 +74,12 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; /** @@ -105,6 +109,8 @@ */ public final class FlattenedFieldMapper extends FieldMapper { + public static final NodeFeature IGNORE_ABOVE_SUPPORT = new NodeFeature("flattened.ignore_above_support"); + public static final String CONTENT_TYPE = "flattened"; public static final String KEYED_FIELD_SUFFIX = "._keyed"; public static final String TIME_SERIES_DIMENSIONS_ARRAY_PARAM = "time_series_dimensions"; @@ -214,7 +220,8 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { meta.get(), splitQueriesOnWhitespace.get(), eagerGlobalOrdinals.get(), - dimensions.get() + dimensions.get(), + ignoreAbove.getValue() ); return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), this); } @@ -642,6 +649,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme private final boolean eagerGlobalOrdinals; private final List dimensions; private final boolean isDimension; + private final int ignoreAbove; public RootFlattenedFieldType( String name, @@ -649,9 +657,10 @@ public RootFlattenedFieldType( boolean hasDocValues, Map meta, boolean splitQueriesOnWhitespace, - boolean eagerGlobalOrdinals + boolean eagerGlobalOrdinals, + int ignoreAbove ) { - this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList()); + this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove); } public RootFlattenedFieldType( @@ -661,7 +670,8 @@ public RootFlattenedFieldType( Map meta, boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, - List dimensions + List dimensions, + int ignoreAbove ) { super( name, @@ -675,6 +685,7 @@ public RootFlattenedFieldType( this.eagerGlobalOrdinals = eagerGlobalOrdinals; this.dimensions = dimensions; this.isDimension = dimensions.isEmpty() == false; + this.ignoreAbove = ignoreAbove; } @Override @@ -708,7 +719,67 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.identity(name(), context, format); + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + } + + private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { + return new SourceValueFetcher(sourcePaths, null) { + @Override + @SuppressWarnings("unchecked") + protected Object parseSourceValue(Object value) { + if (value instanceof Map valueAsMap && valueAsMap.isEmpty() == false) { + final Map result = filterIgnoredValues((Map) valueAsMap); + return result.isEmpty() ? null : result; + } + if (value instanceof String valueAsString && valueAsString.length() <= ignoreAbove) { + return valueAsString; + } + return null; + } + + private Map filterIgnoredValues(final Map values) { + final Map result = new HashMap<>(); + for (final Map.Entry entry : values.entrySet()) { + Object value = filterIgnoredValues(entry.getValue()); + if (value != null) { + result.put(entry.getKey(), value); + } + } + return result; + } + + private Object filterIgnoredValues(final Object entryValue) { + if (entryValue instanceof List valueAsList) { + final List validValues = new ArrayList<>(); + for (Object value : valueAsList) { + if (value instanceof String valueAsString) { + if (valueAsString.length() <= ignoreAbove) { + validValues.add(valueAsString); + } + } else { + validValues.add(value); + } + } + if (validValues.isEmpty()) { + return null; + } + if (validValues.size() == 1) { + // NOTE: for single-value flattened fields do not return an array + return validValues.get(0); + } + return validValues; + } else if (entryValue instanceof String valueAsString) { + if (valueAsString.length() <= ignoreAbove) { + return valueAsString; + } + return null; + } + return entryValue; + } + }; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java index 3f9ba85bee1bb..23d2c4b554d85 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java @@ -76,8 +76,4 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { } } - @Override - public void close() { - // no-op - } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index d1f1e973dc2ff..8d8d0e87cc259 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.ClusterStateTaskExecutor; import org.elasticsearch.cluster.ClusterStateTaskListener; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -270,26 +271,55 @@ public static void resolvePipelinesAndUpdateIndexRequest( final IndexRequest indexRequest, final Metadata metadata ) { - resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis()); + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, Map.of()); + } + + public static void resolvePipelinesAndUpdateIndexRequest( + final DocWriteRequest originalRequest, + final IndexRequest indexRequest, + final Metadata metadata, + Map componentTemplateSubstitutions + ) { + resolvePipelinesAndUpdateIndexRequest( + originalRequest, + indexRequest, + metadata, + System.currentTimeMillis(), + componentTemplateSubstitutions + ); } static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, final Metadata metadata, - final long epochMillis + final long epochMillis, + final Map componentTemplateSubstitutions ) { if (indexRequest.isPipelineResolved()) { return; } - String requestPipeline = indexRequest.getPipeline(); - - Pipelines pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // - .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata)) - .orElse(Pipelines.NO_PIPELINES_DEFINED); + /* + * Here we look for the pipelines associated with the index if the index exists. If the index does not exist we fall back to using + * templates to find the pipelines. But if a user has passed in component template substitutions, they want the settings from those + * used in place of the settings used to create any previous indices. So in that case we use the templates to find the pipelines -- + * we don't fall back to the existing index if we don't find any because it is possible the user has intentionally removed the + * pipeline. + */ + final Pipelines pipelines; + if (componentTemplateSubstitutions.isEmpty()) { + pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // + .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata, Map.of())) + .orElse(Pipelines.NO_PIPELINES_DEFINED); + } else { + pipelines = resolvePipelinesFromIndexTemplates(indexRequest, metadata, componentTemplateSubstitutions).orElse( + Pipelines.NO_PIPELINES_DEFINED + ); + } // The pipeline coming as part of the request always has priority over the resolved one from metadata or templates + String requestPipeline = indexRequest.getPipeline(); if (requestPipeline != null) { indexRequest.setPipeline(requestPipeline); } else { @@ -1443,7 +1473,11 @@ private static Optional resolvePipelinesFromMetadata( return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } - private static Optional resolvePipelinesFromIndexTemplates(IndexRequest indexRequest, Metadata metadata) { + private static Optional resolvePipelinesFromIndexTemplates( + IndexRequest indexRequest, + Metadata metadata, + Map componentTemplateSubstitutions + ) { if (indexRequest.index() == null) { return Optional.empty(); } @@ -1453,7 +1487,7 @@ private static Optional resolvePipelinesFromIndexTemplates(IndexReque // precedence), or if a V2 template does not match, any V1 templates String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false); if (v2Template != null) { - final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template); + final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, componentTemplateSubstitutions); return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index e48677f46d57a..17eda305b5ccf 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -105,19 +105,19 @@ public boolean isContentConsumed() { protected RestRequest( XContentParserConfiguration parserConfig, Map params, - String path, + String rawPath, Map> headers, HttpRequest httpRequest, HttpChannel httpChannel ) { - this(parserConfig, params, path, headers, httpRequest, httpChannel, requestIdGenerator.incrementAndGet()); + this(parserConfig, params, rawPath, headers, httpRequest, httpChannel, requestIdGenerator.incrementAndGet()); } @SuppressWarnings("this-escape") private RestRequest( XContentParserConfiguration parserConfig, Map params, - String path, + String rawPath, Map> headers, HttpRequest httpRequest, HttpChannel httpChannel, @@ -149,7 +149,7 @@ private RestRequest( : parserConfig.withRestApiVersion(effectiveApiVersion); this.httpChannel = httpChannel; this.params = params; - this.rawPath = path; + this.rawPath = rawPath; this.headers = Collections.unmodifiableMap(headers); this.requestId = requestId; } @@ -204,11 +204,10 @@ void ensureSafeBuffers() { */ public static RestRequest request(XContentParserConfiguration parserConfig, HttpRequest httpRequest, HttpChannel httpChannel) { Map params = params(httpRequest.uri()); - String path = path(httpRequest.uri()); return new RestRequest( parserConfig, params, - path, + httpRequest.rawPath(), httpRequest.getHeaders(), httpRequest, httpChannel, @@ -229,15 +228,6 @@ private static Map params(final String uri) { return params; } - private static String path(final String uri) { - final int index = uri.indexOf('?'); - if (index >= 0) { - return uri.substring(0, index); - } else { - return uri; - } - } - /** * Creates a new REST request. The path is not decoded so this constructor will not throw a * {@link BadParameterException}. diff --git a/server/src/main/java/org/elasticsearch/search/MultiValueMode.java b/server/src/main/java/org/elasticsearch/search/MultiValueMode.java index 7f499da9c0fc8..49480816bbbb1 100644 --- a/server/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/server/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -593,12 +593,7 @@ public boolean advanceExact(int parentDoc) throws IOException { return true; } final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); - final int firstChildDoc; - if (childDocs.docID() > prevParentDoc) { - firstChildDoc = childDocs.docID(); - } else { - firstChildDoc = childDocs.advance(prevParentDoc + 1); - } + final int firstChildDoc = getFirstChildDoc(prevParentDoc, childDocs); lastSeenParentDoc = parentDoc; lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren); @@ -700,12 +695,7 @@ public boolean advanceExact(int parentDoc) throws IOException { return true; } final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); - final int firstChildDoc; - if (childDocs.docID() > prevParentDoc) { - firstChildDoc = childDocs.docID(); - } else { - firstChildDoc = childDocs.advance(prevParentDoc + 1); - } + final int firstChildDoc = getFirstChildDoc(prevParentDoc, childDocs); lastSeenParentDoc = parentDoc; lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren); @@ -713,12 +703,22 @@ public boolean advanceExact(int parentDoc) throws IOException { } @Override - public double doubleValue() throws IOException { + public double doubleValue() { return lastEmittedValue; } }; } + private static int getFirstChildDoc(int prevParentDoc, DocIdSetIterator childDocs) throws IOException { + final int firstChildDoc; + if (childDocs.docID() > prevParentDoc) { + firstChildDoc = childDocs.docID(); + } else { + firstChildDoc = childDocs.advance(prevParentDoc + 1); + } + return firstChildDoc; + } + protected double pick( SortedNumericDoubleValues values, double missingValue, @@ -754,7 +754,7 @@ public boolean advanceExact(int target) throws IOException { } @Override - public BytesRef binaryValue() throws IOException { + public BytesRef binaryValue() { return this.value; } }; @@ -768,14 +768,13 @@ public boolean advanceExact(int target) throws IOException { if (values.advanceExact(target)) { value = pick(values); return true; - } else { - value = missingValue; - return missingValue != null; } + value = missingValue; + return missingValue != null; } @Override - public BytesRef binaryValue() throws IOException { + public BytesRef binaryValue() { return value; } }; @@ -825,12 +824,7 @@ public boolean advanceExact(int parentDoc) throws IOException { } final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); - final int firstChildDoc; - if (childDocs.docID() > prevParentDoc) { - firstChildDoc = childDocs.docID(); - } else { - firstChildDoc = childDocs.advance(prevParentDoc + 1); - } + final int firstChildDoc = getFirstChildDoc(prevParentDoc, childDocs); lastSeenParentDoc = parentDoc; lastEmittedValue = pick(selectedValues, builder, childDocs, firstChildDoc, parentDoc, maxChildren); @@ -841,7 +835,7 @@ public boolean advanceExact(int parentDoc) throws IOException { } @Override - public BytesRef binaryValue() throws IOException { + public BytesRef binaryValue() { return lastEmittedValue; } }; @@ -964,12 +958,7 @@ public boolean advanceExact(int parentDoc) throws IOException { } final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); - final int firstChildDoc; - if (childDocs.docID() > prevParentDoc) { - firstChildDoc = childDocs.docID(); - } else { - firstChildDoc = childDocs.advance(prevParentDoc + 1); - } + final int firstChildDoc = getFirstChildDoc(prevParentDoc, childDocs); docID = lastSeenParentDoc = parentDoc; lastEmittedOrd = pick(selectedValues, childDocs, firstChildDoc, parentDoc, maxChildren); diff --git a/server/src/test/java/org/elasticsearch/TransportVersionTests.java b/server/src/test/java/org/elasticsearch/TransportVersionTests.java index f7fc248376e47..e3ab7463ad941 100644 --- a/server/src/test/java/org/elasticsearch/TransportVersionTests.java +++ b/server/src/test/java/org/elasticsearch/TransportVersionTests.java @@ -16,6 +16,7 @@ import java.util.Collections; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -199,4 +200,40 @@ public void testToString() { assertEquals("2000099", TransportVersion.fromId(2_00_00_99).toString()); assertEquals("5000099", TransportVersion.fromId(5_00_00_99).toString()); } + + /** + * Until 9.0 bumps its transport version to 9_000_00_0, all transport changes must be backported to 8.x. + * This test ensures transport versions are dense, so that we have confidence backports have not been missed. + * Note that it does not ensure patches are not missed, but it should catch the majority of misordered + * or missing transport versions. + */ + public void testDenseTransportVersions() { + Set missingVersions = new TreeSet<>(); + TransportVersion previous = null; + for (var tv : TransportVersions.getAllVersions()) { + if (tv.before(TransportVersions.V_8_14_0)) { + continue; + } + if (previous == null) { + previous = tv; + continue; + } + + if (previous.id() + 1000 < tv.id()) { + int nextId = previous.id(); + do { + nextId = (nextId + 1000) / 1000 * 1000; + missingVersions.add(nextId); + } while (nextId + 1000 < tv.id()); + } + previous = tv; + } + if (missingVersions.isEmpty() == false) { + StringBuilder msg = new StringBuilder("Missing transport versions:\n"); + for (Integer id : missingVersions) { + msg.append(" " + id + "\n"); + } + fail(msg.toString()); + } + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index 8f0ff82beab4b..9b1d8c15619ad 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -87,7 +87,8 @@ public Settings getAdditionalIndexSettings( xContentRegistry(), indicesService, systemIndices, - indexSettingsProviders + indexSettingsProviders, + Map.of() ); assertThat(resolvedTemplate.settings().getAsInt("test-setting", -1), is(1)); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index e3c863ee69985..f4e53912d09a7 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -133,7 +133,7 @@ public void tearDown() throws Exception { super.tearDown(); } - public void testIndexData() { + public void testIndexData() throws IOException { Task task = mock(Task.class); // unused BulkRequest bulkRequest = new SimulateBulkRequest(null, null); int bulkItemCount = randomIntBetween(0, 200); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 81c19946753db..bbcf1ca33a0c2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -244,8 +244,8 @@ public void testCreateDataStreamWithFailureStoreInitialized() throws Exception { ActionListener.noop(), true ); - var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); - var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); + var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.startTime()); + var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.startTime()); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); @@ -284,8 +284,8 @@ public void testCreateDataStreamWithFailureStoreUninitialized() throws Exception ActionListener.noop(), false ); - var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); - var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); + var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.startTime()); + var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.startTime()); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); @@ -321,8 +321,8 @@ public void testCreateDataStreamWithFailureStoreWithRefreshRate() throws Excepti ActionListener.noop(), true ); - var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); - var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); + var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.startTime()); + var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.startTime()); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().index(backingIndexName), notNullValue()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java index 796d0892184f5..4f2c84d76b5a4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java @@ -693,10 +693,14 @@ public void testAddAndRemoveAliasClusterStateUpdate() throws Exception { String index = randomAlphaOfLength(5); ClusterState before = createIndex(ClusterState.builder(ClusterName.DEFAULT).build(), index); IndicesAliasesClusterStateUpdateRequest addAliasRequest = new IndicesAliasesClusterStateUpdateRequest( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, List.of(new AliasAction.Add(index, "test", null, null, null, null, null)), List.of(AliasActionResult.buildSuccess(List.of(index), AliasActions.add().aliases("test").indices(index))) ); IndicesAliasesClusterStateUpdateRequest removeAliasRequest = new IndicesAliasesClusterStateUpdateRequest( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, List.of(new AliasAction.Remove(index, "test", true)), List.of(AliasActionResult.buildSuccess(List.of(index), AliasActions.remove().aliases("test").indices(index))) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 38de29010a371..7a1d4b5b1ddf4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1081,7 +1081,7 @@ public void testResolveConflictingMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1143,7 +1143,7 @@ public void testResolveMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1197,6 +1197,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-data-stream-template", + Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1248,7 +1249,12 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex .build(); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries"); + List mappings = MetadataIndexTemplateService.collectMappings( + state, + "timeseries-template", + Map.of(), + "timeseries" + ); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); @@ -1270,6 +1276,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", + Map.of(), DataStream.getDefaultBackingIndexName("timeseries", 1L) ); @@ -1318,6 +1325,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-template", + Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1374,6 +1382,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", + Map.of(), DataStream.getDefaultBackingIndexName("timeseries-template", 1L) ); @@ -2440,7 +2449,12 @@ public void testComposableTemplateWithSubobjectsFalse() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "composable-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "composable-template", "test-index"); + List mappings = MetadataIndexTemplateService.collectMappings( + state, + "composable-template", + Map.of(), + "test-index" + ); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 6ac8e6a853f27..e39ccdf7af5e2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -595,7 +595,32 @@ public void testRoutingPathBooleansInSource() throws IOException { IndexRouting routing = indexRoutingForPath(shards, "foo"); assertIndexShard(routing, Map.of("foo", true), Math.floorMod(hash(List.of("foo", "true")), shards)); assertIndexShard(routing, Map.of("foo", false), Math.floorMod(hash(List.of("foo", "false")), shards)); + } + public void testRoutingPathArraysInSource() throws IOException { + int shards = between(2, 1000); + IndexRouting routing = indexRoutingForPath(shards, "a,b,c,d"); + assertIndexShard( + routing, + Map.of("c", List.of(true), "d", List.of(), "a", List.of("foo", "bar", "foo"), "b", List.of(21, 42)), + // Note that the fields are sorted + Math.floorMod(hash(List.of("a", "foo", "a", "bar", "a", "foo", "b", "21", "b", "42", "c", "true")), shards) + ); + } + + public void testRoutingPathObjectArraysInSource() throws IOException { + int shards = between(2, 1000); + IndexRouting routing = indexRoutingForPath(shards, "a"); + + BytesReference source = source(Map.of("a", List.of("foo", Map.of("foo", "bar")))); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source, s -> {}) + ); + assertThat( + e.getMessage(), + equalTo("Error extracting routing: Failed to parse object: expecting value token but found [START_OBJECT]") + ); } public void testRoutingPathBwc() throws IOException { @@ -668,7 +693,11 @@ private void assertIndexShard(IndexRouting routing, Map source, IndexRouting.ExtractFromSource.Builder b = r.builder(); for (Map.Entry e : flattened.entrySet()) { - b.addMatching(e.getKey(), new BytesRef(e.getValue().toString())); + if (e.getValue() instanceof List listValue) { + listValue.forEach(v -> b.addMatching(e.getKey(), new BytesRef(v.toString()))); + } else { + b.addMatching(e.getKey(), new BytesRef(e.getValue().toString())); + } } String idFromBuilder = b.createId(suffix, () -> { throw new AssertionError(); }); assertThat(idFromBuilder, equalTo(idFromSource)); diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java index 46e34ddffb6c4..fd3495e9d24aa 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterName; @@ -26,7 +27,6 @@ import org.elasticsearch.cluster.LocalMasterServiceTask; import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.SimpleBatchedExecutor; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.coordination.ClusterStatePublisher; import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; @@ -1570,7 +1570,7 @@ public void onAckFailure(Exception e) { masterService.submitUnbatchedStateUpdateTask( "test2", - new AckedClusterStateUpdateTask(ackedRequest(TimeValue.ZERO, null), null) { + new AckedClusterStateUpdateTask(ackedRequest(TimeValue.ZERO, TimeValue.MINUS_ONE), null) { @Override public ClusterState execute(ClusterState currentState) { return ClusterState.builder(currentState).build(); @@ -1622,7 +1622,7 @@ public void onAckTimeout() { masterService.submitUnbatchedStateUpdateTask( "test2", - new AckedClusterStateUpdateTask(ackedRequest(ackTimeout, null), null) { + new AckedClusterStateUpdateTask(ackedRequest(ackTimeout, TimeValue.MINUS_ONE), null) { @Override public ClusterState execute(ClusterState currentState) { threadPool.getThreadContext().addResponseHeader(responseHeaderName, responseHeaderValue); @@ -1677,7 +1677,7 @@ public void onAckTimeout() { masterService.submitUnbatchedStateUpdateTask( "test2", - new AckedClusterStateUpdateTask(ackedRequest(TimeValue.MINUS_ONE, null), null) { + new AckedClusterStateUpdateTask(ackedRequest(TimeValue.MINUS_ONE, TimeValue.MINUS_ONE), null) { @Override public ClusterState execute(ClusterState currentState) { return ClusterState.builder(currentState).build(); @@ -2656,20 +2656,15 @@ public static ClusterState discoveryState(MasterService masterService) { } /** - * Returns a plain {@link AckedRequest} that does not implement any functionality outside of the timeout getters. + * Returns a plain {@link AcknowledgedRequest} that does not implement any functionality outside of the timeout getters. */ - public static AckedRequest ackedRequest(TimeValue ackTimeout, TimeValue masterNodeTimeout) { - return new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return ackTimeout; - } - - @Override - public TimeValue masterNodeTimeout() { - return masterNodeTimeout; + public static AcknowledgedRequest ackedRequest(TimeValue ackTimeout, TimeValue masterNodeTimeout) { + class BareAcknowledgedRequest extends AcknowledgedRequest { + BareAcknowledgedRequest() { + super(masterNodeTimeout, ackTimeout); } - }; + } + return new BareAcknowledgedRequest(); } /** diff --git a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java index 76add3fea71c3..dfe3cf10fd494 100644 --- a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java @@ -25,6 +25,7 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoField; +import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalAccessor; import java.util.List; import java.util.Locale; @@ -85,38 +86,32 @@ private void assertDateMathEquals(String text, String expected, String pattern) } private void assertDateMathEquals(String text, String expected, String pattern, Locale locale) { - long gotMillisJava = dateMathToMillis(text, DateFormatter.forPattern(pattern), locale); - long expectedMillis = DateFormatters.from(DateFormatter.forPattern("strict_date_optional_time").withLocale(locale).parse(expected)) - .toInstant() - .toEpochMilli(); + Instant gotInstant = dateMathToInstant(text, DateFormatter.forPattern(pattern), locale).truncatedTo(ChronoUnit.MILLIS); + Instant expectedInstant = DateFormatters.from( + DateFormatter.forPattern("strict_date_optional_time").withLocale(locale).parse(expected) + ).toInstant().truncatedTo(ChronoUnit.MILLIS); - assertThat(gotMillisJava, equalTo(expectedMillis)); + assertThat(gotInstant, equalTo(expectedInstant)); } public void testWeekBasedDates() { - // as per WeekFields.ISO first week starts on Monday and has minimum 4 days + // the years and weeks this outputs depends on where the first day of the first week is for each year DateFormatter dateFormatter = DateFormatters.forPattern("YYYY-ww"); - // first week of 2016 starts on Monday 2016-01-04 as previous week in 2016 has only 3 days assertThat( - DateFormatters.from(dateFormatter.parse("2016-01")), - equalTo(ZonedDateTime.of(2016, 01, 04, 0, 0, 0, 0, ZoneOffset.UTC)) + DateFormatters.from(dateFormatter.parse("2016-02")), + equalTo(ZonedDateTime.of(2016, 01, 03, 0, 0, 0, 0, ZoneOffset.UTC)) ); - // first week of 2015 starts on Monday 2014-12-29 because 4days belong to 2019 assertThat( - DateFormatters.from(dateFormatter.parse("2015-01")), - equalTo(ZonedDateTime.of(2014, 12, 29, 0, 0, 0, 0, ZoneOffset.UTC)) + DateFormatters.from(dateFormatter.parse("2015-02")), + equalTo(ZonedDateTime.of(2015, 01, 04, 0, 0, 0, 0, ZoneOffset.UTC)) ); - // as per WeekFields.ISO first week starts on Monday and has minimum 4 days dateFormatter = DateFormatters.forPattern("YYYY"); - // first week of 2016 starts on Monday 2016-01-04 as previous week in 2016 has only 3 days - assertThat(DateFormatters.from(dateFormatter.parse("2016")), equalTo(ZonedDateTime.of(2016, 01, 04, 0, 0, 0, 0, ZoneOffset.UTC))); - - // first week of 2015 starts on Monday 2014-12-29 because 4days belong to 2019 - assertThat(DateFormatters.from(dateFormatter.parse("2015")), equalTo(ZonedDateTime.of(2014, 12, 29, 0, 0, 0, 0, ZoneOffset.UTC))); + assertThat(DateFormatters.from(dateFormatter.parse("2016")), equalTo(ZonedDateTime.of(2015, 12, 27, 0, 0, 0, 0, ZoneOffset.UTC))); + assertThat(DateFormatters.from(dateFormatter.parse("2015")), equalTo(ZonedDateTime.of(2014, 12, 28, 0, 0, 0, 0, ZoneOffset.UTC))); } public void testEpochMillisParser() { @@ -600,8 +595,8 @@ public void testYearWithoutMonthRoundUp() { assertDateMathEquals("1500", "1500-01-01T23:59:59.999", "uuuu"); assertDateMathEquals("2022", "2022-01-01T23:59:59.999", "uuuu"); assertDateMathEquals("2022", "2022-01-01T23:59:59.999", "yyyy"); - // cannot reliably default week based years due to locale changing. See JavaDateFormatter javadocs - assertDateMathEquals("2022", "2022-01-03T23:59:59.999", "YYYY", Locale.ROOT); + // weird locales can change this to epoch-based + assertDateMathEquals("2022", "2021-12-26T23:59:59.999", "YYYY", Locale.ROOT); } private void assertRoundupFormatter(String format, String input, long expectedMilliSeconds) { @@ -789,30 +784,28 @@ public void testExceptionWhenCompositeParsingFailsDateMath() { String text = "2014-06-06T12:01:02.123"; ElasticsearchParseException e1 = expectThrows( ElasticsearchParseException.class, - () -> dateMathToMillis(text, DateFormatter.forPattern(pattern), randomLocale(random())) + () -> dateMathToInstant(text, DateFormatter.forPattern(pattern), randomLocale(random())) ); assertThat(e1.getMessage(), containsString(pattern)); assertThat(e1.getMessage(), containsString(text)); } - private long dateMathToMillis(String text, DateFormatter dateFormatter, Locale locale) { + private Instant dateMathToInstant(String text, DateFormatter dateFormatter, Locale locale) { DateFormatter javaFormatter = dateFormatter.withLocale(locale); DateMathParser javaDateMath = javaFormatter.toDateMathParser(); - return javaDateMath.parse(text, () -> 0, true, (ZoneId) null).toEpochMilli(); + return javaDateMath.parse(text, () -> 0, true, null); } public void testDayOfWeek() { - // 7 (ok joda) vs 1 (java by default) but 7 with customized org.elasticsearch.common.time.IsoLocale.ISO8601 ZonedDateTime now = LocalDateTime.of(2009, 11, 15, 1, 32, 8, 328402).atZone(ZoneOffset.UTC); // Sunday DateFormatter javaFormatter = DateFormatter.forPattern("8e").withZone(ZoneOffset.UTC); - assertThat(javaFormatter.format(now), equalTo("7")); + assertThat(javaFormatter.format(now), equalTo("1")); } public void testStartOfWeek() { - // 2019-21 (ok joda) vs 2019-22 (java by default) but 2019-21 with customized org.elasticsearch.common.time.IsoLocale.ISO8601 ZonedDateTime now = LocalDateTime.of(2019, 5, 26, 1, 32, 8, 328402).atZone(ZoneOffset.UTC); DateFormatter javaFormatter = DateFormatter.forPattern("8YYYY-ww").withZone(ZoneOffset.UTC); - assertThat(javaFormatter.format(now), equalTo("2019-21")); + assertThat(javaFormatter.format(now), equalTo("2019-22")); } // these parsers should allow both ',' and '.' as a decimal point diff --git a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java index 2e8394f920029..cd386d9159aa6 100644 --- a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java @@ -19,6 +19,7 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatterBuilder; import java.time.format.ResolverStyle; +import java.time.temporal.ChronoUnit; import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.LongSupplier; @@ -82,12 +83,12 @@ public void testOverridingLocaleOrZoneAndCompositeRoundUpParser() { // the pattern has to be composite and the match should not be on the first one DateFormatter formatter = DateFormatter.forPattern("date||epoch_millis").withLocale(randomLocale(random())); DateMathParser parser = formatter.toDateMathParser(); - long gotMillis = parser.parse("297276785531", () -> 0, true, (ZoneId) null).toEpochMilli(); + Instant gotMillis = parser.parse("297276785531", () -> 0, true, null).truncatedTo(ChronoUnit.MILLIS); assertDateEquals(gotMillis, "297276785531", "297276785531"); formatter = DateFormatter.forPattern("date||epoch_millis").withZone(ZoneOffset.UTC); parser = formatter.toDateMathParser(); - gotMillis = parser.parse("297276785531", () -> 0, true, (ZoneId) null).toEpochMilli(); + gotMillis = parser.parse("297276785531", () -> 0, true, null).truncatedTo(ChronoUnit.MILLIS); assertDateEquals(gotMillis, "297276785531", "297276785531"); } @@ -99,7 +100,7 @@ public void testWeekBasedDate() { // defaulting missing day of week formatter = DateFormatter.forPattern("YYYY'W'ww[e]");// YYYY'W'wwe // second week of 2022 is starting on Monday 10th Jan - assertDateMathEquals(formatter.toDateMathParser(), "2022W02", "2022-01-10T23:59:59.999Z", 0, true, ZoneOffset.UTC); + assertDateMathEquals(formatter.toDateMathParser(), "2022W02", "2022-01-02T23:59:59.999Z", 0, true, ZoneOffset.UTC); } public void testDayOfYear() { @@ -125,16 +126,16 @@ public void testAMPMWithTimeMissing() { public void testWeekDates() { DateFormatter formatter = DateFormatter.forPattern("YYYY-ww"); - assertDateMathEquals(formatter.toDateMathParser(), "2016-01", "2016-01-04T23:59:59.999Z", 0, true, ZoneOffset.UTC); + assertDateMathEquals(formatter.toDateMathParser(), "2016-02", "2016-01-03T23:59:59.999Z", 0, true, ZoneOffset.UTC); formatter = DateFormatter.forPattern("YYYY"); - assertDateMathEquals(formatter.toDateMathParser(), "2016", "2016-01-04T23:59:59.999Z", 0, true, ZoneOffset.UTC); + assertDateMathEquals(formatter.toDateMathParser(), "2016", "2015-12-27T23:59:59.999Z", 0, true, ZoneOffset.UTC); formatter = DateFormatter.forPattern("YYYY-ww"); - assertDateMathEquals(formatter.toDateMathParser(), "2015-01", "2014-12-29T23:59:59.999Z", 0, true, ZoneOffset.UTC); + assertDateMathEquals(formatter.toDateMathParser(), "2015-02", "2015-01-04T23:59:59.999Z", 0, true, ZoneOffset.UTC); formatter = DateFormatter.forPattern("YYYY"); - assertDateMathEquals(formatter.toDateMathParser(), "2015", "2014-12-29T23:59:59.999Z", 0, true, ZoneOffset.UTC); + assertDateMathEquals(formatter.toDateMathParser(), "2015", "2014-12-28T23:59:59.999Z", 0, true, ZoneOffset.UTC); } public void testBasicDates() { @@ -343,7 +344,7 @@ public void testTimestamps() { // also check other time units DateMathParser parser = DateFormatter.forPattern("epoch_second||date_optional_time").toDateMathParser(); - long datetime = parser.parse("1418248078", () -> 0).toEpochMilli(); + Instant datetime = parser.parse("1418248078", () -> 0).truncatedTo(ChronoUnit.MILLIS); assertDateEquals(datetime, "1418248078", "2014-12-10T21:47:58.000"); // for date_optional_time a timestamp with more than 9digits is epoch @@ -401,14 +402,14 @@ private void assertDateMathEquals( boolean roundUp, ZoneId timeZone ) { - long gotMillis = parser.parse(toTest, () -> now, roundUp, timeZone).toEpochMilli(); + Instant gotMillis = parser.parse(toTest, () -> now, roundUp, timeZone).truncatedTo(ChronoUnit.MILLIS); assertDateEquals(gotMillis, toTest, expected); } - private void assertDateEquals(long gotMillis, String original, String expected) { - long expectedMillis = parser.parse(expected, () -> 0).toEpochMilli(); - if (gotMillis != expectedMillis) { - ZonedDateTime zonedDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(gotMillis), ZoneOffset.UTC); + private void assertDateEquals(Instant gotMillis, String original, String expected) { + Instant expectedMillis = parser.parse(expected, () -> 0).truncatedTo(ChronoUnit.MILLIS); + if (gotMillis.equals(expectedMillis) == false) { + ZonedDateTime zonedDateTime = ZonedDateTime.ofInstant(gotMillis, ZoneOffset.UTC); fail(Strings.format(""" Date math not equal Original : %s diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index b0a9a9e9fe7dc..83553503c3c5e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -268,8 +268,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimension", true); }), IndexMode.TIME_SERIES); - Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> b.array("field", true, false)))); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field", true, false); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 0e97d2b3b46ae..eaa7bf6528203 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -1518,6 +1518,37 @@ public void testStoredNestedSubObjectWithNameOverlappingParentName() throws IOEx {"path":{"at":{"foo":"A"}}}""", syntheticSource); } + public void testCopyToLogicInsideObject() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path"); + b.startObject("properties"); + { + b.startObject("at").field("type", "keyword").field("copy_to", "copy_top.copy").endObject(); + } + b.endObject(); + b.endObject(); + b.startObject("copy_top"); + b.startObject("properties"); + { + b.startObject("copy").field("type", "keyword").endObject(); + } + b.endObject(); + b.endObject(); + })).documentMapper(); + + CheckedConsumer document = b -> { + b.startObject("path"); + b.field("at", "A"); + b.endObject(); + }; + + var doc = documentMapper.parse(source(document)); + assertNotNull(doc.docs().get(0).getField("copy_top.copy")); + + var syntheticSource = syntheticSource(documentMapper, document); + assertEquals("{\"path\":{\"at\":\"A\"}}", syntheticSource); + } + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) throws IOException { // We exclude ignored source field since in some cases it contains an exact copy of a part of document source. diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index f92df7a421cf9..1b8a2d68cd930 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -266,11 +266,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimension", true); }), IndexMode.TIME_SERIES); - Exception e = expectThrows( - DocumentParsingException.class, - () -> mapper.parse(source(b -> b.array("field", "192.168.1.1", "192.168.1.1"))) - ); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field", "192.168.1.1", "192.168.1.1"); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 2d2fad23b3831..5b218fb077d32 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -384,8 +384,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimension", true); }), IndexMode.TIME_SERIES); - Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> b.array("field", "1234", "45678")))); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field", "1234", "45678"); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 8ba57824cf434..3312c94e8a0e1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -811,4 +811,40 @@ public void testFlattenExplicitSubobjectsTrue() { exception.getMessage() ); } + + public void testFindParentMapper() { + MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); + + var rootBuilder = new RootObjectMapper.Builder("_doc", Optional.empty()); + rootBuilder.add(new KeywordFieldMapper.Builder("keyword", IndexVersion.current())); + + var child = new ObjectMapper.Builder("child", Optional.empty()); + child.add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())); + child.add(new KeywordFieldMapper.Builder("keyword.with.dot", IndexVersion.current())); + var secondLevelChild = new ObjectMapper.Builder("child2", Optional.empty()); + secondLevelChild.add(new KeywordFieldMapper.Builder("keyword22", IndexVersion.current())); + child.add(secondLevelChild); + rootBuilder.add(child); + + var childWithDot = new ObjectMapper.Builder("childwith.dot", Optional.empty()); + childWithDot.add(new KeywordFieldMapper.Builder("keyword3", IndexVersion.current())); + childWithDot.add(new KeywordFieldMapper.Builder("keyword4.with.dot", IndexVersion.current())); + rootBuilder.add(childWithDot); + + RootObjectMapper root = rootBuilder.build(rootContext); + + assertEquals("_doc", root.findParentMapper("keyword").fullPath()); + assertNull(root.findParentMapper("aa")); + + assertEquals("child", root.findParentMapper("child.keyword2").fullPath()); + assertEquals("child", root.findParentMapper("child.keyword.with.dot").fullPath()); + assertNull(root.findParentMapper("child.long")); + assertNull(root.findParentMapper("child.long.hello")); + assertEquals("child.child2", root.findParentMapper("child.child2.keyword22").fullPath()); + + assertEquals("childwith.dot", root.findParentMapper("childwith.dot.keyword3").fullPath()); + assertEquals("childwith.dot", root.findParentMapper("childwith.dot.keyword4.with.dot").fullPath()); + assertNull(root.findParentMapper("childwith.dot.long")); + assertNull(root.findParentMapper("childwith.dot.long.hello")); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index b27b8bd9766f3..9d56938f185de 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -28,10 +28,13 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -63,13 +66,19 @@ protected IndexVersion getVersion() { } private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder mappings) throws IOException { + return createDocumentMapper(getVersion(), routingPath, mappings); + } + + private DocumentMapper createDocumentMapper(IndexVersion version, String routingPath, XContentBuilder mappings) throws IOException { return createMapperService( + version, getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 200) // Allow tests that use many dimensions .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-10-29T00:00:00Z") .build(), + () -> true, mappings ).documentMapper(); } @@ -644,6 +653,44 @@ public void testDifferentDimensions() throws IOException { assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); } + public void testMultiValueDimensionsNotSupportedBeforeTsidHashing() throws IOException { + IndexVersion priorToTsidHashing = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_ID_HASHING); + DocumentMapper docMapper = createDocumentMapper( + priorToTsidHashing, + "a", + mapping(b -> b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject()) + ); + + String a1 = randomAlphaOfLength(10); + String a2 = randomAlphaOfLength(10); + CheckedConsumer fields = d -> d.field("a", new String[] { a1, a2 }); + DocumentParsingException exception = assertThrows(DocumentParsingException.class, () -> parseDocument(docMapper, fields)); + assertThat(exception.getMessage(), containsString("Dimension field [a] cannot be a multi-valued field")); + } + + public void testMultiValueDimensions() throws IOException { + DocumentMapper docMapper = createDocumentMapper( + IndexVersions.TIME_SERIES_ID_HASHING, + "a", + mapping(b -> b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject()) + ); + + String a1 = randomAlphaOfLength(10); + String a2 = randomAlphaOfLength(10); + List docs = List.of( + parseDocument(docMapper, d -> d.field("a", new String[] { a1 })), + parseDocument(docMapper, d -> d.field("a", new String[] { a1, a2 })), + parseDocument(docMapper, d -> d.field("a", new String[] { a2, a1 })), + parseDocument(docMapper, d -> d.field("a", new String[] { a1, a2, a1 })), + parseDocument(docMapper, d -> d.field("a", new String[] { a2, a1, a2 })) + ); + List tsids = docs.stream() + .map(doc -> doc.rootDoc().getBinaryValue("_tsid").toString()) + .distinct() + .collect(Collectors.toList()); + assertThat(tsids, hasSize(docs.size())); + } + /** * Documents with fewer dimensions have a different value. */ diff --git a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java index 4de02178beec1..f4e114da1fa51 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java @@ -240,64 +240,6 @@ private void testWriteMergedWithMixedValues(Object value, List multipleV assertEquals(expected, map.get("foo")); } - public void testWriteMergedWithVoidValue() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - XContentDataHelper.writeMerged(destination, "field", List.of(XContentDataHelper.nothing())); - - destination.endObject(); - - assertEquals("{}", Strings.toString(destination)); - } - - public void testWriteMergedWithMultipleVoidValues() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - XContentDataHelper.writeMerged( - destination, - "field", - List.of(XContentDataHelper.nothing(), XContentDataHelper.nothing(), XContentDataHelper.nothing()) - ); - - destination.endObject(); - - assertEquals("{}", Strings.toString(destination)); - } - - public void testWriteMergedWithMixedVoidValues() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - var value = XContentFactory.contentBuilder(XContentType.JSON).value(34); - XContentDataHelper.writeMerged( - destination, - "field", - List.of(XContentDataHelper.nothing(), XContentDataHelper.encodeXContentBuilder(value), XContentDataHelper.nothing()) - ); - - destination.endObject(); - - assertEquals("{\"field\":34}", Strings.toString(destination)); - } - - public void testWriteMergedWithArraysAndVoidValues() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - var value = XContentFactory.contentBuilder(XContentType.JSON).value(List.of(3, 4)); - XContentDataHelper.writeMerged( - destination, - "field", - List.of(XContentDataHelper.nothing(), XContentDataHelper.encodeXContentBuilder(value), XContentDataHelper.nothing()) - ); - - destination.endObject(); - - assertEquals("{\"field\":[3,4]}", Strings.toString(destination)); - } - private Map executeWriteMergedOnRepeated(Object value) throws IOException { return executeWriteMergedOnTwoEncodedValues(value, value); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 2f245a319f8cc..285431b881add 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; import org.elasticsearch.xcontent.XContentBuilder; @@ -200,11 +201,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimensions", List.of("key1", "key2", "field3.key3")); }), IndexMode.TIME_SERIES); - Exception e = expectThrows( - DocumentParsingException.class, - () -> mapper.parse(source(b -> b.array("field.key1", "value1", "value2"))) - ); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field.key1] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field.key1", "value1", "value2"); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java index cadc836f7c4ea..f494af259c504 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java @@ -165,10 +165,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() { - // Nothing to do. - } } private static final ToScriptFieldFactory MOCK_TO_SCRIPT_FIELD = (dv, n) -> new DelegateDocValuesField( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index e70b7f2091ad2..7873458eb46ee 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; @@ -32,12 +33,12 @@ public class RootFlattenedFieldTypeTests extends FieldTypeTestCase { - private static RootFlattenedFieldType createDefaultFieldType() { - return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false); + private static RootFlattenedFieldType createDefaultFieldType(int ignoreAbove) { + return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, ignoreAbove); } public void testValueForDisplay() { - RootFlattenedFieldType ft = createDefaultFieldType(); + RootFlattenedFieldType ft = createDefaultFieldType(Integer.MAX_VALUE); String fieldValue = "{ \"key\": \"value\" }"; BytesRef storedValue = new BytesRef(fieldValue); @@ -45,7 +46,7 @@ public void testValueForDisplay() { } public void testTermQuery() { - RootFlattenedFieldType ft = createDefaultFieldType(); + RootFlattenedFieldType ft = createDefaultFieldType(Integer.MAX_VALUE); Query expected = new TermQuery(new Term("field", "value")); assertEquals(expected, ft.termQuery("value", null)); @@ -53,21 +54,45 @@ public void testTermQuery() { expected = AutomatonQueries.caseInsensitiveTermQuery(new Term("field", "Value")); assertEquals(expected, ft.termQueryCaseInsensitive("Value", null)); - RootFlattenedFieldType unsearchable = new RootFlattenedFieldType("field", false, true, Collections.emptyMap(), false, false); + RootFlattenedFieldType unsearchable = new RootFlattenedFieldType( + "field", + false, + true, + Collections.emptyMap(), + false, + false, + Integer.MAX_VALUE + ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } public void testExistsQuery() { - RootFlattenedFieldType ft = new RootFlattenedFieldType("field", true, false, Collections.emptyMap(), false, false); + RootFlattenedFieldType ft = new RootFlattenedFieldType( + "field", + true, + false, + Collections.emptyMap(), + false, + false, + Integer.MAX_VALUE + ); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null)); - RootFlattenedFieldType withDv = new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false); + RootFlattenedFieldType withDv = new RootFlattenedFieldType( + "field", + true, + true, + Collections.emptyMap(), + false, + false, + Integer.MAX_VALUE + ); assertEquals(new FieldExistsQuery("field"), withDv.existsQuery(null)); } public void testFuzzyQuery() { - RootFlattenedFieldType ft = createDefaultFieldType(); + RootFlattenedFieldType ft = createDefaultFieldType(Integer.MAX_VALUE); Query expected = new FuzzyQuery(new Term("field", "value"), 2, 1, 50, true); Query actual = ft.fuzzyQuery("value", Fuzziness.fromEdits(2), 1, 50, true, MOCK_CONTEXT); @@ -88,7 +113,7 @@ public void testFuzzyQuery() { } public void testRangeQuery() { - RootFlattenedFieldType ft = createDefaultFieldType(); + RootFlattenedFieldType ft = createDefaultFieldType(Integer.MAX_VALUE); TermRangeQuery expected = new TermRangeQuery("field", new BytesRef("lower"), new BytesRef("upper"), false, false); assertEquals(expected, ft.rangeQuery("lower", "upper", false, false, MOCK_CONTEXT)); @@ -107,7 +132,7 @@ public void testRangeQuery() { } public void testRegexpQuery() { - RootFlattenedFieldType ft = createDefaultFieldType(); + RootFlattenedFieldType ft = createDefaultFieldType(Integer.MAX_VALUE); Query expected = new RegexpQuery(new Term("field", "val.*")); Query actual = ft.regexpQuery("val.*", 0, 0, 10, null, MOCK_CONTEXT); @@ -121,7 +146,7 @@ public void testRegexpQuery() { } public void testWildcardQuery() { - RootFlattenedFieldType ft = createDefaultFieldType(); + RootFlattenedFieldType ft = createDefaultFieldType(Integer.MAX_VALUE); Query expected = new WildcardQuery(new Term("field", new BytesRef("valu*"))); assertEquals(expected, ft.wildcardQuery("valu*", null, MOCK_CONTEXT)); @@ -135,9 +160,61 @@ public void testWildcardQuery() { public void testFetchSourceValue() throws IOException { Map sourceValue = Map.of("key", "value"); - RootFlattenedFieldType ft = createDefaultFieldType(); + RootFlattenedFieldType ft = createDefaultFieldType(Integer.MAX_VALUE); assertEquals(List.of(sourceValue), fetchSourceValue(ft, sourceValue)); assertEquals(List.of(), fetchSourceValue(ft, null)); } + + public void testFetchSourceValueWithIgnoreAbove() throws IOException { + Map sourceValue = Map.of("key", "test ignore above"); + + assertEquals(List.of(), fetchSourceValue(createDefaultFieldType(10), sourceValue)); + assertEquals(List.of(sourceValue), fetchSourceValue(createDefaultFieldType(20), sourceValue)); + } + + public void testFetchSourceValueWithList() throws IOException { + Map sourceValue = Map.of("key1", List.of("one", "two", "three")); + + assertEquals(List.of(Map.of("key1", List.of("one", "two"))), fetchSourceValue(createDefaultFieldType(3), sourceValue)); + } + + public void testFetchSourceValueWithMultipleFields() throws IOException { + Map sourceValue = Map.of( + "key1", + "test", + "key2", + List.of("one", "two", "three"), + "key3", + "hi", + "key4", + List.of("the quick brown fox", "jumps over the lazy dog") + ); + + assertEquals( + List.of(Map.of("key2", List.of("one", "two"), "key3", "hi")), + fetchSourceValue(createDefaultFieldType(3), sourceValue) + ); + } + + public void testFetchSourceValueWithMixedFieldTypes() throws IOException { + Map sourceValue = Map.of("key1", List.of("one", 1, "two", 2)); + + assertEquals(List.of(Map.of("key1", List.of("one", 1, "two", 2))), fetchSourceValue(createDefaultFieldType(3), sourceValue)); + } + + public void testFetchSourceValueWithNonString() throws IOException { + Map sourceValue = Map.of("key1", List.of(100, 200), "key2", 50L, "key3", new Tuple<>(10, 100)); + + assertEquals( + List.of(Map.of("key1", List.of(100, 200), "key2", 50L, "key3", new Tuple<>(10, 100))), + fetchSourceValue(createDefaultFieldType(3), sourceValue) + ); + } + + public void testFetchSourceValueFilterStringsOnly() throws IOException { + Map sourceValue = Map.of("key1", List.of("the quick brown", 1_234_567, "jumps over", 2_456)); + + assertEquals(List.of(Map.of("key1", List.of(1_234_567, 2_456))), fetchSourceValue(createDefaultFieldType(8), sourceValue)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java index e9a47668faca5..946dbea45b4ba 100644 --- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java @@ -126,8 +126,6 @@ public Collection getChildResources() { throw new UnsupportedOperationException(UNSUPPORTED); } - @Override - public void close() {} }; } @@ -229,8 +227,6 @@ public Collection getChildResources() { throw new UnsupportedOperationException(UNSUPPORTED); } - @Override - public void close() {} }; } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 799ea5c0a1c74..a8f556649cb8c 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -32,16 +32,20 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils; import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.Maps; @@ -73,6 +77,7 @@ import org.mockito.ArgumentMatcher; import org.mockito.invocation.InvocationOnMock; +import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -2586,7 +2591,7 @@ public void testResolveFinalPipelineWithDateMathExpression() { // index name matches with IDM: IndexRequest indexRequest = new IndexRequest(""); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis, Map.of()); assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); @@ -2916,6 +2921,83 @@ public void testResolvePipelinesWithNonePipeline() { } } + public void testResolvePipelinesAndUpdateIndexRequestWithComponentTemplateSubstitutions() throws IOException { + final String componentTemplateName = "test-component-template"; + final String indexName = "my-index-1"; + final String indexPipeline = "index-pipeline"; + final String realTemplatePipeline = "template-pipeline"; + final String substitutePipeline = "substitute-pipeline"; + + Metadata metadata; + { + // Build up cluster state metadata + IndexMetadata.Builder builder = IndexMetadata.builder(indexName) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0); + ComponentTemplate realComponentTemplate = new ComponentTemplate( + new Template( + Settings.builder().put("index.default_pipeline", realTemplatePipeline).build(), + CompressedXContent.fromJSON("{}"), + null + ), + null, + null + ); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-index-*")) + .componentTemplates(List.of(componentTemplateName)) + .build(); + metadata = Metadata.builder() + .put(builder) + .indexTemplates(Map.of("my-index-template", composableIndexTemplate)) + .componentTemplates(Map.of("test-component-template", realComponentTemplate)) + .build(); + } + + Map componentTemplateSubstitutions; + { + ComponentTemplate simulatedComponentTemplate = new ComponentTemplate( + new Template( + Settings.builder().put("index.default_pipeline", substitutePipeline).build(), + CompressedXContent.fromJSON("{}"), + null + ), + null, + null + ); + componentTemplateSubstitutions = Map.of(componentTemplateName, simulatedComponentTemplate); + } + + { + /* + * Here there is a pipeline in the request. This takes precedence over anything in the index or templates or component template + * substitutions. + */ + IndexRequest indexRequest = new IndexRequest(indexName).setPipeline(indexPipeline); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); + assertThat(indexRequest.getPipeline(), equalTo(indexPipeline)); + } + { + /* + * Here there is no pipeline in the request, but there is one in the substitute component template. So it takes precedence. + */ + IndexRequest indexRequest = new IndexRequest(indexName); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); + assertThat(indexRequest.getPipeline(), equalTo(substitutePipeline)); + } + { + /* + * This one is tricky. Since the index exists and there are no component template substitutions, we're going to use the actual + * index in this case rather than its template. The index does not have a default pipeline set, so it's "_none" instead of + * realTemplatePipeline. + */ + IndexRequest indexRequest = new IndexRequest(indexName); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, Map.of()); + assertThat(indexRequest.getPipeline(), equalTo("_none")); + } + } + private static Tuple randomMapEntry() { return tuple(randomAlphaOfLength(5), randomObject()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java index bb17983264072..a297f5d13254b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java @@ -80,11 +80,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimension", true); }), IndexMode.TIME_SERIES); - Exception e = expectThrows( - DocumentParsingException.class, - () -> mapper.parse(source(b -> b.array("field", randomNumber(), randomNumber(), randomNumber()))) - ); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field", randomNumber(), randomNumber(), randomNumber()); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index ce843fc3e15ee..81e120511a40f 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -104,9 +104,12 @@ public DataSourceResponse.ObjectArrayGenerator accept(DataSourceHandler handler) } } - record LeafMappingParametersGenerator(String fieldName, FieldType fieldType, Set eligibleCopyToFields) - implements - DataSourceRequest { + record LeafMappingParametersGenerator( + String fieldName, + FieldType fieldType, + Set eligibleCopyToFields, + DynamicMapping dynamicMapping + ) implements DataSourceRequest { public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandler handler) { return handler.handle(this); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 86cf071ce8696..89850cd56bbd0 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -10,6 +10,7 @@ package org.elasticsearch.logsdb.datageneration.datasource; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -48,7 +49,11 @@ private Supplier> keywordMapping( // We only add copy_to to keywords because we get into trouble with numeric fields that are copied to dynamic fields. // If first copied value is numeric, dynamic field is created with numeric field type and then copy of text values fail. // Actual value being copied does not influence the core logic of copy_to anyway. - if (ESTestCase.randomDouble() <= 0.05) { + // + // TODO + // We don't use copy_to on fields that are inside an object with dynamic: strict + // because we'll hit https://github.com/elastic/elasticsearch/issues/113049. + if (request.dynamicMapping() != DynamicMapping.FORBIDDEN && ESTestCase.randomDouble() <= 0.05) { var options = request.eligibleCopyToFields() .stream() .filter(f -> f.equals(request.fieldName()) == false) diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java index c6d9d94d61892..ba03b2f91c53c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java @@ -62,7 +62,8 @@ List generateChildFields(DynamicMapping dynamicMapping) { new DataSourceRequest.LeafMappingParametersGenerator( fieldName, fieldTypeInfo.fieldType(), - context.getEligibleCopyToDestinations() + context.getEligibleCopyToDestinations(), + dynamicMapping ) ); var generator = fieldTypeInfo.fieldType() diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java index 7b1bf4a77dc72..57e3ce3ce2a86 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java @@ -21,7 +21,7 @@ public interface PredefinedField { FieldDataGenerator generator(DataSource dataSource); - record WithType(String fieldName, FieldType fieldType) implements PredefinedField { + record WithType(String fieldName, FieldType fieldType, DynamicMapping dynamicMapping) implements PredefinedField { @Override public String name() { return fieldName; @@ -31,7 +31,7 @@ public String name() { public FieldDataGenerator generator(DataSource dataSource) { // copy_to currently not supported for predefined fields, use WithGenerator if needed var mappingParametersGenerator = dataSource.get( - new DataSourceRequest.LeafMappingParametersGenerator(fieldName, fieldType, Set.of()) + new DataSourceRequest.LeafMappingParametersGenerator(fieldName, fieldType, Set.of(), dynamicMapping) ); return fieldType().generator(fieldName, dataSource, mappingParametersGenerator); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 0940b3ef96ddd..d597d7b59f240 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -233,10 +233,6 @@ public long ramBytesUsed() { return 0; // Unknown } - @Override - public void close() { - - } }; } diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 09ffae9b24be1..0b307d9dc04cc 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -77,6 +77,9 @@ tasks.named("yamlRestTestV7CompatTest").configure { 'ml/evaluate_data_frame/Test outlier_detection auc_roc include curve', 'ml/evaluate_data_frame/Test classification auc_roc', 'ml/evaluate_data_frame/Test classification auc_roc with default top_classes_field', + 'security/authz/13_index_datemath/Test indexing documents with datemath, when forbidden', + 'security/authz/13_index_datemath/Test indexing documents with datemath, when permitted', + 'security/authz/13_index_datemath/Test bulk indexing with datemath when only some are allowed', ].join(',') } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java index 1072e6ee4c899..53247d6428bfb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java @@ -9,11 +9,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Priority; import org.elasticsearch.core.Nullable; @@ -36,7 +36,7 @@ public class OperationModeUpdateTask extends ClusterStateUpdateTask { public static AckedClusterStateUpdateTask wrap( OperationModeUpdateTask task, - AckedRequest request, + AcknowledgedRequest request, ActionListener listener ) { return new AckedClusterStateUpdateTask(task.priority(), request, listener) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java index 175f550ab5d3b..9338077a55570 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java @@ -50,6 +50,10 @@ public class AggregationOperator implements Operator { * Nanoseconds this operator has spent running the aggregations. */ private long aggregationNanos; + /** + * Nanoseconds this operator has spent running the aggregations final evaluation. + */ + private long aggregationFinishNanos; /** * Count of pages this operator has processed. */ @@ -117,6 +121,7 @@ public void finish() { if (finished) { return; } + long start = System.nanoTime(); finished = true; Block[] blocks = null; boolean success = false; @@ -136,6 +141,7 @@ public void finish() { if (success == false && blocks != null) { Releasables.closeExpectNoException(blocks); } + aggregationFinishNanos += System.nanoTime() - start; } } @@ -175,7 +181,7 @@ public String toString() { @Override public Operator.Status status() { - return new Status(aggregationNanos, pagesProcessed); + return new Status(aggregationNanos, aggregationFinishNanos, pagesProcessed); } public static class Status implements Operator.Status { @@ -189,6 +195,11 @@ public static class Status implements Operator.Status { * Nanoseconds this operator has spent running the aggregations. */ private final long aggregationNanos; + + /** + * Nanoseconds this operator has spent running the aggregations final evaluation. + */ + private final Long aggregationFinishNanos; /** * Count of pages this operator has processed. */ @@ -197,21 +208,31 @@ public static class Status implements Operator.Status { /** * Build. * @param aggregationNanos Nanoseconds this operator has spent running the aggregations. + * @param aggregationFinishNanos Nanoseconds this operator has spent running the aggregations. * @param pagesProcessed Count of pages this operator has processed. */ - public Status(long aggregationNanos, int pagesProcessed) { + public Status(long aggregationNanos, long aggregationFinishNanos, int pagesProcessed) { this.aggregationNanos = aggregationNanos; + this.aggregationFinishNanos = aggregationFinishNanos; this.pagesProcessed = pagesProcessed; } protected Status(StreamInput in) throws IOException { aggregationNanos = in.readVLong(); + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_AGGREGATION_OPERATOR_STATUS_FINISH_NANOS)) { + aggregationFinishNanos = in.readOptionalVLong(); + } else { + aggregationFinishNanos = null; + } pagesProcessed = in.readVInt(); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeVLong(aggregationNanos); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_AGGREGATION_OPERATOR_STATUS_FINISH_NANOS)) { + out.writeOptionalVLong(aggregationFinishNanos); + } out.writeVInt(pagesProcessed); } @@ -227,6 +248,13 @@ public long aggregationNanos() { return aggregationNanos; } + /** + * Nanoseconds this operator has spent running the aggregations final evaluation. + */ + public long aggregationFinishNanos() { + return aggregationFinishNanos; + } + /** * Count of pages this operator has processed. */ @@ -241,6 +269,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (builder.humanReadable()) { builder.field("aggregation_time", TimeValue.timeValueNanos(aggregationNanos)); } + builder.field("aggregation_finish_nanos", aggregationFinishNanos); + if (builder.humanReadable()) { + builder.field( + "aggregation_finish_time", + aggregationFinishNanos == null ? null : TimeValue.timeValueNanos(aggregationFinishNanos) + ); + } builder.field("pages_processed", pagesProcessed); return builder.endObject(); @@ -251,12 +286,14 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Status status = (Status) o; - return aggregationNanos == status.aggregationNanos && pagesProcessed == status.pagesProcessed; + return aggregationNanos == status.aggregationNanos + && pagesProcessed == status.pagesProcessed + && Objects.equals(aggregationFinishNanos, status.aggregationFinishNanos); } @Override public int hashCode() { - return Objects.hash(aggregationNanos, pagesProcessed); + return Objects.hash(aggregationNanos, aggregationFinishNanos, pagesProcessed); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java index b5ae35bfc8d7f..5e0e625abb914 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java @@ -46,6 +46,7 @@ import java.util.Objects; import java.util.function.IntFunction; import java.util.function.Supplier; +import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; @@ -325,7 +326,11 @@ private static void checkState(boolean condition, String msg) { @Override public String toString() { - return this.getClass().getSimpleName() + "[" + "aggregators=" + aggregatorFactories + "]"; + String aggregatorDescriptions = aggregatorFactories.stream() + .map(factory -> "\"" + factory.describe() + "\"") + .collect(Collectors.joining(", ")); + + return this.getClass().getSimpleName() + "[" + "aggregators=[" + aggregatorDescriptions + "]]"; } record SegmentID(int shardIndex, int segmentIndex) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AggregationOperatorStatusTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AggregationOperatorStatusTests.java index 5d17538ee85ae..f9d806b72cb46 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AggregationOperatorStatusTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AggregationOperatorStatusTests.java @@ -16,7 +16,7 @@ public class AggregationOperatorStatusTests extends AbstractWireSerializingTestCase { public static AggregationOperator.Status simple() { - return new AggregationOperator.Status(200012, 123); + return new AggregationOperator.Status(200012, 400036, 123); } public static String simpleToJson() { @@ -24,6 +24,8 @@ public static String simpleToJson() { { "aggregation_nanos" : 200012, "aggregation_time" : "200micros", + "aggregation_finish_nanos" : 400036, + "aggregation_finish_time" : "400micros", "pages_processed" : 123 }"""; } @@ -39,18 +41,20 @@ protected Writeable.Reader instanceReader() { @Override public AggregationOperator.Status createTestInstance() { - return new AggregationOperator.Status(randomNonNegativeLong(), randomNonNegativeInt()); + return new AggregationOperator.Status(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeInt()); } @Override protected AggregationOperator.Status mutateInstance(AggregationOperator.Status instance) { long aggregationNanos = instance.aggregationNanos(); + long aggregationFinishNanos = instance.aggregationFinishNanos(); int pagesProcessed = instance.pagesProcessed(); - switch (between(0, 1)) { + switch (between(0, 2)) { case 0 -> aggregationNanos = randomValueOtherThan(aggregationNanos, ESTestCase::randomNonNegativeLong); - case 1 -> pagesProcessed = randomValueOtherThan(pagesProcessed, ESTestCase::randomNonNegativeInt); + case 1 -> aggregationFinishNanos = randomValueOtherThan(aggregationFinishNanos, ESTestCase::randomNonNegativeLong); + case 2 -> pagesProcessed = randomValueOtherThan(pagesProcessed, ESTestCase::randomNonNegativeInt); default -> throw new UnsupportedOperationException(); } - return new AggregationOperator.Status(aggregationNanos, pagesProcessed); + return new AggregationOperator.Status(aggregationNanos, aggregationFinishNanos, pagesProcessed); } } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 44550c62bd7c5..cf5b3453fa97c 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -323,6 +323,35 @@ public void testProfile() throws IOException { ); } + public void testProfileOrdinalsGroupingOperator() throws IOException { + indexTimestampData(1); + + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | STATS AVG(value) BY test.keyword"); + builder.profile(true); + if (Build.current().isSnapshot()) { + // Lock to shard level partitioning, so we get consistent profile output + builder.pragmas(Settings.builder().put("data_partitioning", "shard").build()); + } + Map result = runEsql(builder); + + List> signatures = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); + for (Map p : profiles) { + fixTypesOnProfile(p); + assertThat(p, commonProfile()); + List sig = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> operators = (List>) p.get("operators"); + for (Map o : operators) { + sig.add((String) o.get("operator")); + } + signatures.add(sig); + } + + assertThat(signatures.get(0).get(2), equalTo("OrdinalsGroupingOperator[aggregators=[\"sum of longs\", \"count\"]]")); + } + public void testInlineStatsProfile() throws IOException { assumeTrue("INLINESTATS only available on snapshots", Build.current().isSnapshot()); indexTimestampData(1); @@ -533,7 +562,9 @@ private String checkOperatorProfile(Map o) { .entry("processing_nanos", greaterThan(0)) .entry("processed_queries", List.of("*:*")); case "ValuesSourceReaderOperator" -> basicProfile().entry("readers_built", matchesMap().extraOk()); - case "AggregationOperator" -> matchesMap().entry("pages_processed", greaterThan(0)).entry("aggregation_nanos", greaterThan(0)); + case "AggregationOperator" -> matchesMap().entry("pages_processed", greaterThan(0)) + .entry("aggregation_nanos", greaterThan(0)) + .entry("aggregation_finish_nanos", greaterThan(0)); case "ExchangeSinkOperator" -> matchesMap().entry("pages_accepted", greaterThan(0)); case "ExchangeSourceOperator" -> matchesMap().entry("pages_emitted", greaterThan(0)).entry("pages_waiting", 0); case "ProjectOperator", "EvalOperator" -> basicProfile(); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 4ec15eb94464c..f539644bfedff 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -45,7 +45,7 @@ emp_no:integer | x:date evalDateFormat -from employees | sort hire_date | eval x = date_format(hire_date), y = date_format("YYYY-MM-dd", hire_date) | keep emp_no, x, y | limit 5; +from employees | sort hire_date | eval x = date_format(hire_date), y = date_format("yyyy-MM-dd", hire_date) | keep emp_no, x, y | limit 5; emp_no:integer | x:keyword | y:keyword 10009 | 1985-02-18T00:00:00.000Z | 1985-02-18 @@ -367,6 +367,80 @@ date1:date | dd_ms:integer 2023-12-02T11:00:00.000Z | 1000 ; +evalDateDiffMonthAsWhole0Months + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME +| EVAL msecs=DATE_DIFF("milliseconds", from, to), months=DATE_DIFF("month", from, to) +; + + from:date | to:date | msecs:integer| months:integer +2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|1 |0 + +; + +evalDateDiffMonthAsWhole1Month + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-02-01T00:00:00"::DATETIME +| EVAL secs=DATE_DIFF("seconds", from, to), months=DATE_DIFF("month", from, to) +; + + from:date | to:date | secs:integer| months:integer +2023-12-31T23:59:59.999Z|2024-02-01T00:00:00.000Z|2678400 |1 + +; + +evalDateDiffYearAsWhole0Years +required_capability: date_diff_year_calendarial + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME +| EVAL msecs=DATE_DIFF("milliseconds", from, to), years=DATE_DIFF("year", from, to) +; + + from:date | to:date | msecs:integer | years:integer +2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|1 |0 +; + +evalDateDiffYearAsWhole1Year +required_capability: date_diff_year_calendarial + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME +| EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to) +; + + from:date | to:date | secs:integer| years:integer +2023-12-31T23:59:59.999Z|2025-01-01T00:00:00.000Z|31622400 |1 +; + +evalDateDiffYearAsWhole1Year +required_capability: date_diff_year_calendarial + +ROW from="2024-01-01T00:00:00Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME +| EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to) +; + + from:date | to:date | secs:integer| years:integer +2024-01-01T00:00:00.000Z|2025-01-01T00:00:00.000Z|31622400 |1 +; + +evalDateDiffYearForDocs +required_capability: date_diff_year_calendarial + +// tag::evalDateDiffYearForDocs[] +ROW end_23="2023-12-31T23:59:59.999Z"::DATETIME, + start_24="2024-01-01T00:00:00.000Z"::DATETIME, + end_24="2024-12-31T23:59:59.999"::DATETIME +| EVAL end23_to_start24=DATE_DIFF("year", end_23, start_24) +| EVAL end23_to_end24=DATE_DIFF("year", end_23, end_24) +| EVAL start_to_end_24=DATE_DIFF("year", start_24, end_24) +// end::evalDateDiffYearForDocs[] +; + +// tag::evalDateDiffYearForDocs-result[] + end_23:date | start_24:date | end_24:date |end23_to_start24:integer|end23_to_end24:integer|start_to_end_24:integer +2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|2024-12-31T23:59:59.999Z|0 |1 |0 +// end::evalDateDiffYearForDocs-result[] +; + evalDateParseWithSimpleDate row a = "2023-02-01" | eval b = date_parse("yyyy-MM-dd", a) | keep b; @@ -882,7 +956,7 @@ docsDateFormat // tag::docsDateFormat[] FROM employees | KEEP first_name, last_name, hire_date -| EVAL hired = DATE_FORMAT("YYYY-MM-dd", hire_date) +| EVAL hired = DATE_FORMAT("yyyy-MM-dd", hire_date) // end::docsDateFormat[] | SORT first_name | LIMIT 3 @@ -900,7 +974,7 @@ evalDateFormatString required_capability: string_literal_auto_casting ROW a = 1 -| EVAL df = DATE_FORMAT("YYYY-MM-dd", "1989-06-02T00:00:00.000Z") +| EVAL df = DATE_FORMAT("yyyy-MM-dd", "1989-06-02T00:00:00.000Z") ; a:integer | df:keyword diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec new file mode 100644 index 0000000000000..2f6313925032e --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec @@ -0,0 +1,116 @@ +############################################### +# Tests for QSTR function +# + +qstrWithField +required_capability: qstr_function + +// tag::qstr-with-field[] +from books +| where qstr("author: Faulkner") +| keep book_no, author +| sort book_no +| limit 5; +// end::qstr-with-field[] + +// tag::qstr-with-field-result[] +book_no:keyword | author:text +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] +2713 | William Faulkner +2847 | Colleen Faulkner +2883 | William Faulkner +3293 | Danny Faulkner +; +// end::qstr-with-field-result[] + +qstrWithMultipleFields +required_capability: qstr_function + +from books +| where qstr("title:Return* AND author:*Tolkien") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +qstrWithQueryExpressions +required_capability: qstr_function + +from books +| where qstr(CONCAT("title:Return*", " AND author:*Tolkien")) +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +7350 | Return of the Shadow +; + +qstrWithDisjunction +required_capability: qstr_function + +from books +| where qstr("title:Return") or year > 2020 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2714 | Return of the King Being the Third Part of The Lord of the Rings +6818 | Hadji Murad +7350 | Return of the Shadow +; + +qstrWithConjunction +required_capability: qstr_function + +from books +| where qstr("title: Rings") and ratings > 4.6 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 |A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 |The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; + +qstrWithFunctionPushedToLucene +required_capability: qstr_function + +from hosts +| where qstr("host: beta") and cidr_match(ip1, "127.0.0.2/32", "127.0.0.3/32") +| keep card, host, ip0, ip1; +ignoreOrder:true + +card:keyword |host:keyword |ip0:ip |ip1:ip +eth1 |beta |127.0.0.1 |127.0.0.2 +; + +qstrWithFunctionNotPushedToLucene +required_capability: qstr_function + +from books +| where qstr("title: rings") and length(description) > 600 +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +2675 | The Lord of the Rings - Boxed Set +2714 | Return of the King Being the Third Part of The Lord of the Rings +; + +qstrWithMultipleWhereClauses +required_capability: qstr_function + +from books +| where qstr("title: rings") +| where qstr("year: [1 TO 2005]") +| keep book_no, title; +ignoreOrder:true + +book_no:keyword | title:text +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings +7140 | The Lord of the Rings Poster Collection: Six Paintings by Alan Lee (No. 1) +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 02a2cac0513c0..16c19083f78be 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -1176,7 +1176,7 @@ avg_lang:double | max_lang:integer docsStatsGroupByMultipleValues // tag::statsGroupByMultipleValues[] FROM employees -| EVAL hired = DATE_FORMAT("YYYY", hire_date) +| EVAL hired = DATE_FORMAT("yyyy", hire_date) | STATS avg_salary = AVG(salary) BY hired, languages.long | EVAL avg_salary = ROUND(avg_salary) | SORT hired, languages.long diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java new file mode 100644 index 0000000000000..e6f11ca1f44d2 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringFunctionIT.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plugin; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; +import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; +import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.junit.Before; + +import java.util.List; + +import static org.elasticsearch.test.ListMatcher.matchesList; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class QueryStringFunctionIT extends AbstractEsqlIntegTestCase { + + @Before + public void setupIndex() { + createAndPopulateIndex(); + } + + @Override + protected EsqlQueryResponse run(EsqlQueryRequest request) { + assumeTrue("qstr function available in snapshot builds only", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + return super.run(request); + } + + public void testSimpleQueryString() { + var query = """ + FROM test + | WHERE qstr("content: dog") + | KEEP id + | SORT id + """; + + try (var resp = run(query)) { + assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of("id"))); + assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of("INTEGER"))); + // values + List> values = getValuesList(resp); + assertMap(values, matchesList().item(List.of(1)).item(List.of(3)).item(List.of(4)).item(List.of(5))); + } + } + + public void testMultiFieldQueryString() { + var query = """ + FROM test + | WHERE qstr("dog OR canine") + | KEEP id + """; + + try (var resp = run(query)) { + assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of("id"))); + assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of("INTEGER"))); + // values + List> values = getValuesList(resp); + assertThat(values.size(), equalTo(5)); + } + } + + public void testQueryStringWithinEval() { + var query = """ + FROM test + | EVAL matches_query = qstr("title: fox") + """; + + var error = expectThrows(VerificationException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("[QSTR] function is only supported in WHERE commands")); + } + + public void testInvalidQueryStringEof() { + var query = """ + FROM test + | WHERE qstr("content: ((((dog") + """; + + var error = expectThrows(QueryShardException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("Failed to parse query [content: ((((dog]")); + assertThat(error.getRootCause().getMessage(), containsString("Encountered \"\" at line 1, column 16")); + } + + public void testInvalidQueryStringLexicalError() { + var query = """ + FROM test + | WHERE qstr("/") + """; + + var error = expectThrows(QueryShardException.class, () -> run(query)); + assertThat(error.getMessage(), containsString("Failed to parse query [/]")); + assertThat( + error.getRootCause().getMessage(), + containsString("Lexical error at line 1, column 2. Encountered: (in lexical state 2)") + ); + } + + private void createAndPopulateIndex() { + var indexName = "test"; + var client = client().admin().indices(); + var CreateRequest = client.prepareCreate(indexName) + .setSettings(Settings.builder().put("index.number_of_shards", 1)) + .setMapping("id", "type=integer", "content", "type=text"); + assertAcked(CreateRequest); + client().prepareBulk() + .add( + new IndexRequest(indexName).id("1") + .source("id", 1, "content", "The quick brown animal swiftly jumps over a lazy dog", "title", "A Swift Fox's Journey") + ) + .add( + new IndexRequest(indexName).id("2") + .source("id", 2, "content", "A speedy brown fox hops effortlessly over a sluggish canine", "title", "The Fox's Leap") + ) + .add( + new IndexRequest(indexName).id("3") + .source("id", 3, "content", "Quick and nimble, the fox vaults over the lazy dog", "title", "Brown Fox in Action") + ) + .add( + new IndexRequest(indexName).id("4") + .source( + "id", + 4, + "content", + "A fox that is quick and brown jumps over a dog that is quite lazy", + "title", + "Speedy Animals" + ) + ) + .add( + new IndexRequest(indexName).id("5") + .source( + "id", + 5, + "content", + "With agility, a quick brown fox bounds over a slow-moving dog", + "title", + "Foxes and Canines" + ) + ) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + ensureYellow(indexName); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index a722d33054795..597c349273eb2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -311,11 +311,21 @@ public enum Cap { */ CATEGORIZE(true), + /** + * QSTR function + */ + QSTR_FUNCTION(true), + /** * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. * https://github.com/elastic/elasticsearch/issues/112704 */ - FIXED_WRONG_IS_NOT_NULL_CHECK_ON_CASE; + FIXED_WRONG_IS_NOT_NULL_CHECK_ON_CASE, + + /** + * Compute year differences in full calendar years. + */ + DATE_DIFF_YEAR_CALENDARIAL; private final boolean snapshotOnly; private final FeatureFlag featureFlag; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 9714d3fce6d9f..c466f9ebb5e53 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -187,6 +188,7 @@ else if (p instanceof Lookup lookup) { checkFilterMatchConditions(p, failures); checkMatchCommand(p, failures); + checkFullTextQueryFunctions(p, failures); }); checkRemoteEnrich(plan, failures); @@ -657,4 +659,31 @@ private static void checkMatchCommand(LogicalPlan plan, Set failures) { } } } + + private static void checkFullTextQueryFunctions(LogicalPlan plan, Set failures) { + if (plan instanceof Filter f) { + Expression condition = f.condition(); + if (condition instanceof FullTextFunction ftf) { + // Similar to cases present in org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineFilters - + // we can't check if it can be pushed down as we don't have yet information about the fields present in the + // StringQueryPredicate + plan.forEachDown(LogicalPlan.class, lp -> { + if ((lp instanceof Filter || lp instanceof OrderBy || lp instanceof EsRelation) == false) { + failures.add( + fail( + plan, + "[{}] function cannot be used after {}", + ftf.functionName(), + lp.sourceText().split(" ")[0].toUpperCase(Locale.ROOT) + ) + ); + } + }); + } + } else { + plan.forEachExpression(FullTextFunction.class, ftf -> { + failures.add(fail(ftf, "[{}] function is only supported in WHERE commands", ftf.functionName())); + }); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index ce8c20e4fbf11..5a6430e0fdfad 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Top; import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import org.elasticsearch.xpack.esql.expression.function.aggregate.WeightedAvg; +import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryStringFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; @@ -386,8 +387,10 @@ private FunctionDefinition[][] functions() { private static FunctionDefinition[][] snapshotFunctions() { return new FunctionDefinition[][] { new FunctionDefinition[] { + def(Rate.class, Rate::withUnresolvedTimestamp, "rate"), def(Categorize.class, Categorize::new, "categorize"), - def(Rate.class, Rate::withUnresolvedTimestamp, "rate") } }; + // Full text functions + def(QueryStringFunction.class, QueryStringFunction::new, "qstr") } }; } public EsqlFunctionRegistry snapshotRegistry() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java new file mode 100644 index 0000000000000..54730eec4f317 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.function.Function; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNullAndFoldable; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +/** + * Base class for full-text functions that use ES queries to match documents. + * These functions needs to be pushed down to Lucene queries to be executed - there's no Evaluator for them, but depend on + * {@link org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer} to rewrite them into Lucene queries. + */ +public abstract class FullTextFunction extends Function { + public static List getNamedWriteables() { + List entries = new ArrayList<>(); + if (EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()) { + entries.add(QueryStringFunction.ENTRY); + } + return entries; + } + + private final Expression query; + + protected FullTextFunction(Source source, Expression query) { + super(source, singletonList(query)); + this.query = query; + } + + protected FullTextFunction(StreamInput in) throws IOException { + this(Source.readFrom((StreamInput & PlanStreamInput) in), in.readNamedWriteable(Expression.class)); + } + + @Override + public DataType dataType() { + return DataType.BOOLEAN; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + return isString(query(), sourceText(), DEFAULT).and(isNotNullAndFoldable(query(), functionName(), DEFAULT)); + } + + public Expression query() { + return query; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(query); + } + + public abstract Query asQuery(); +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java new file mode 100644 index 0000000000000..fa331acd08655 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunction.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Full text function that performs a {@link QueryStringQuery} . + */ +public class QueryStringFunction extends FullTextFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "QStr", + QueryStringFunction::new + ); + + @FunctionInfo( + returnType = "boolean", + preview = true, + description = "Performs a query string query. Returns true if the provided query string matches the row.", + examples = { @Example(file = "qstr-function", tag = "qstr-with-field") } + ) + public QueryStringFunction( + Source source, + @Param( + name = "query", + type = { "keyword", "text" }, + description = "Query string in Lucene query string format." + ) Expression queryString + ) { + super(source, queryString); + } + + private QueryStringFunction(StreamInput in) throws IOException { + super(in); + } + + @Override + public String functionName() { + return "QSTR"; + } + + @Override + public Query asQuery() { + Object queryAsObject = query().fold(); + if (queryAsObject instanceof BytesRef queryAsBytesRef) { + return new QueryStringQuery(source(), queryAsBytesRef.utf8ToString(), Map.of(), null); + } else { + throw new IllegalArgumentException("Query in QSTR needs to be resolved to a string"); + } + } + + @Override + public Expression replaceChildren(List newChildren) { + return new QueryStringFunction(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, QueryStringFunction::new, query()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java index 582785d023945..f9039417e48a6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java @@ -66,7 +66,7 @@ public class DateDiff extends EsqlScalarFunction { */ public enum Part implements DateTimeField { - YEAR((start, end) -> end.getYear() - start.getYear(), "years", "yyyy", "yy"), + YEAR((start, end) -> safeToInt(ChronoUnit.YEARS.between(start, end)), "years", "yyyy", "yy"), QUARTER((start, end) -> safeToInt(IsoFields.QUARTER_YEARS.between(start, end)), "quarters", "qq", "q"), MONTH((start, end) -> safeToInt(ChronoUnit.MONTHS.between(start, end)), "months", "mm", "m"), DAYOFYEAR((start, end) -> safeToInt(ChronoUnit.DAYS.between(start, end)), "dy", "y"), @@ -126,36 +126,44 @@ public static Part resolve(String dateTimeUnit) { } } - @FunctionInfo(returnType = "integer", description = """ - Subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of `unit`. - If `startTimestamp` is later than the `endTimestamp`, negative values are returned.""", detailedDescription = """ - [cols=\"^,^\",role=\"styled\"] - |=== - 2+h|Datetime difference units - - s|unit - s|abbreviations - - | year | years, yy, yyyy - | quarter | quarters, qq, q - | month | months, mm, m - | dayofyear | dy, y - | day | days, dd, d - | week | weeks, wk, ww - | weekday | weekdays, dw - | hour | hours, hh - | minute | minutes, mi, n - | second | seconds, ss, s - | millisecond | milliseconds, ms - | microsecond | microseconds, mcs - | nanosecond | nanoseconds, ns - |=== - - Note that while there is an overlap between the function's supported units and - {esql}'s supported time span literals, these sets are distinct and not - interchangeable. Similarly, the supported abbreviations are conveniently shared - with implementations of this function in other established products and not - necessarily common with the date-time nomenclature used by {es}.""", examples = @Example(file = "date", tag = "docsDateDiff")) + @FunctionInfo( + returnType = "integer", + description = """ + Subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of `unit`. + If `startTimestamp` is later than the `endTimestamp`, negative values are returned.""", + detailedDescription = """ + [cols=\"^,^\",role=\"styled\"] + |=== + 2+h|Datetime difference units + + s|unit + s|abbreviations + + | year | years, yy, yyyy + | quarter | quarters, qq, q + | month | months, mm, m + | dayofyear | dy, y + | day | days, dd, d + | week | weeks, wk, ww + | weekday | weekdays, dw + | hour | hours, hh + | minute | minutes, mi, n + | second | seconds, ss, s + | millisecond | milliseconds, ms + | microsecond | microseconds, mcs + | nanosecond | nanoseconds, ns + |=== + + Note that while there is an overlap between the function's supported units and + {esql}'s supported time span literals, these sets are distinct and not + interchangeable. Similarly, the supported abbreviations are conveniently shared + with implementations of this function in other established products and not + necessarily common with the date-time nomenclature used by {es}.""", + examples = { @Example(file = "date", tag = "docsDateDiff"), @Example(description = """ + When subtracting in calendar units - like year, month a.s.o. - only the fully elapsed units are counted. + To avoid this and obtain also remainders, simply switch to the next smaller unit and do the date math accordingly. + """, file = "date", tag = "evalDateDiffYearForDocs") } + ) public DateDiff( Source source, @Param(name = "unit", type = { "keyword", "text" }, description = "Time difference unit") Expression unit, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index f991429651c76..0a71bce2575fa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.core.util.Queries; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -194,6 +195,8 @@ public static boolean canPushToSource(Expression exp, Predicate return mqp.field() instanceof FieldAttribute && DataType.isString(mqp.field().dataType()); } else if (exp instanceof StringQueryPredicate) { return true; + } else if (exp instanceof FullTextFunction) { + return true; } return false; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index b508dc6556456..18aa2628fdc7c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Check; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils; @@ -84,9 +85,17 @@ public final class EsqlExpressionTranslators { new ExpressionTranslators.StringQueries(), new ExpressionTranslators.Matches(), new ExpressionTranslators.MultiMatches(), + new FullTextFunctions(), new Scalars() ); + public static class FullTextFunctions extends ExpressionTranslator { + @Override + protected Query asQuery(FullTextFunction fullTextFunction, TranslatorHandler handler) { + return fullTextFunction.asQuery(); + } + } + public static Query toQuery(Expression e, TranslatorHandler handler) { Query translation = null; for (ExpressionTranslator translator : QUERY_TRANSLATORS) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 315309fbad677..9b4d51af244b8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -65,6 +65,7 @@ import org.elasticsearch.xpack.esql.execution.PlanExecutor; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -201,6 +202,7 @@ public List getNamedWriteables() { entries.addAll(EsqlScalarFunction.getNamedWriteables()); entries.addAll(AggregateFunction.getNamedWriteables()); entries.addAll(LogicalPlan.getNamedWriteables()); + entries.addAll(FullTextFunction.getNamedWriteables()); entries.addAll(PhysicalPlan.getNamedWriteables()); return entries; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index faf9d04532f1a..3e8d1e4e71562 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -251,6 +251,10 @@ public final void test() throws Throwable { "can't use match command in csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_COMMAND.capabilityName()) ); + assumeFalse( + "can't use QSTR function in csv tests", + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.QSTR_FUNCTION.capabilityName()) + ); if (Build.current().isSnapshot()) { assertThat( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java index fe74883a0c24f..69a68af91f5db 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/SerializationTestUtils.java @@ -29,6 +29,7 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; @@ -123,6 +124,7 @@ public static NamedWriteableRegistry writableRegistry() { entries.addAll(AggregateFunction.getNamedWriteables()); entries.addAll(Block.getNamedWriteables()); entries.addAll(LogicalPlan.getNamedWriteables()); + entries.addAll(FullTextFunction.getNamedWriteables()); entries.addAll(PhysicalPlan.getNamedWriteables()); return new NamedWriteableRegistry(entries); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index f8a48d0fd7b4c..0b83b76992546 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1107,6 +1107,86 @@ private void assertMatchCommand(String lineAndColumn, String command, String que assertThat(error(query, defaultAnalyzer, exception), containsString(expectedErrorMessage)); } + public void testQueryStringFunctionsNotAllowedAfterCommands() throws Exception { + assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + + // Source commands + assertEquals("1:13: [QSTR] function cannot be used after SHOW", error("show info | where qstr(\"8.16.0\")")); + assertEquals("1:17: [QSTR] function cannot be used after ROW", error("row a= \"Anna\" | where qstr(\"Anna\")")); + + // Processing commands + assertEquals( + "1:43: [QSTR] function cannot be used after DISSECT", + error("from test | dissect first_name \"%{foo}\" | where qstr(\"Connection\")") + ); + assertEquals("1:27: [QSTR] function cannot be used after DROP", error("from test | drop emp_no | where qstr(\"Anna\")")); + assertEquals( + "1:71: [QSTR] function cannot be used after ENRICH", + error("from test | enrich languages on languages with lang = language_name | where qstr(\"Anna\")") + ); + assertEquals("1:26: [QSTR] function cannot be used after EVAL", error("from test | eval z = 2 | where qstr(\"Anna\")")); + assertEquals( + "1:44: [QSTR] function cannot be used after GROK", + error("from test | grok last_name \"%{WORD:foo}\" | where qstr(\"Anna\")") + ); + assertEquals("1:27: [QSTR] function cannot be used after KEEP", error("from test | keep emp_no | where qstr(\"Anna\")")); + assertEquals("1:24: [QSTR] function cannot be used after LIMIT", error("from test | limit 10 | where qstr(\"Anna\")")); + assertEquals( + "1:35: [QSTR] function cannot be used after MV_EXPAND", + error("from test | mv_expand last_name | where qstr(\"Anna\")") + ); + assertEquals( + "1:45: [QSTR] function cannot be used after RENAME", + error("from test | rename last_name as full_name | where qstr(\"Anna\")") + ); + assertEquals( + "1:52: [QSTR] function cannot be used after STATS", + error("from test | STATS c = COUNT(emp_no) BY languages | where qstr(\"Anna\")") + ); + + // Some combination of processing commands + assertEquals( + "1:38: [QSTR] function cannot be used after LIMIT", + error("from test | keep emp_no | limit 10 | where qstr(\"Anna\")") + ); + assertEquals( + "1:46: [QSTR] function cannot be used after MV_EXPAND", + error("from test | limit 10 | mv_expand last_name | where qstr(\"Anna\")") + ); + assertEquals( + "1:52: [QSTR] function cannot be used after KEEP", + error("from test | mv_expand last_name | keep last_name | where qstr(\"Anna\")") + ); + assertEquals( + "1:77: [QSTR] function cannot be used after RENAME", + error("from test | STATS c = COUNT(emp_no) BY languages | rename c as total_emps | where qstr(\"Anna\")") + ); + assertEquals( + "1:54: [QSTR] function cannot be used after KEEP", + error("from test | rename last_name as name | keep emp_no | where qstr(\"Anna\")") + ); + } + + public void testQueryStringFunctionsOnlyAllowedInWhere() throws Exception { + assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + + assertEquals("1:22: [QSTR] function is only supported in WHERE commands", error("from test | eval y = qstr(\"Anna\")")); + assertEquals("1:18: [QSTR] function is only supported in WHERE commands", error("from test | sort qstr(\"Connection\") asc")); + assertEquals("1:5: [QSTR] function is only supported in WHERE commands", error("row qstr(\"Connection\")")); + assertEquals( + "1:23: [QSTR] function is only supported in WHERE commands", + error("from test | STATS c = qstr(\"foo\") BY languages") + ); + } + + public void testQueryStringFunctionArgNotNullOrConstant() throws Exception { + assumeTrue("skipping because QSTR is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + + assertEquals("1:19: argument of [QSTR] must be a constant, received [first_name]", error("from test | where qstr(first_name)")); + assertEquals("1:19: argument of [QSTR] cannot be null, received [null]", error("from test | where qstr(null)")); + // Other value types are tested in QueryStringFunctionTests + } + public void testCoalesceWithMixedNumericTypes() { assertEquals( "1:22: second argument of [coalesce(languages, height)] must be [integer], found value [height] type [double]", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java index ab20a5ce0cc6b..a2aa447c748e9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.plan.AbstractNodeSerializationTests; @@ -33,6 +34,7 @@ protected final NamedWriteableRegistry getNamedWriteableRegistry() { entries.addAll(Attribute.getNamedWriteables()); entries.addAll(EsqlScalarFunction.getNamedWriteables()); entries.addAll(AggregateFunction.getNamedWriteables()); + entries.addAll(FullTextFunction.getNamedWriteables()); entries.add(UnsupportedAttribute.ENTRY); entries.add(UnsupportedAttribute.NAMED_EXPRESSION_ENTRY); entries.add(UnsupportedAttribute.EXPRESSION_ENTRY); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java new file mode 100644 index 0000000000000..e622ff5ba2579 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringFunctionTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.fulltext; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +@FunctionName("qstr") +public class QueryStringFunctionTests extends AbstractFunctionTestCase { + + public QueryStringFunctionTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new LinkedList<>(); + for (DataType strType : Arrays.stream(DataType.values()).filter(DataType::isString).toList()) { + suppliers.add( + new TestCaseSupplier( + "<" + strType + ">", + List.of(strType), + () -> testCase(strType, randomAlphaOfLengthBetween(1, 10), equalTo(true)) + ) + ); + } + List errorsSuppliers = errorsForCasesWithoutExamples(suppliers, (v, p) -> "string"); + // Don't test null, as it is not allowed but the expected message is not a type error - so we check it separately in VerifierTests + return parameterSuppliersFromTypedData(errorsSuppliers.stream().filter(s -> s.types().contains(DataType.NULL) == false).toList()); + } + + public final void testFold() { + Expression expression = buildLiteralExpression(testCase); + if (testCase.getExpectedTypeError() != null) { + assertTypeResolutionFailure(expression); + return; + } + assertFalse("expected resolved", expression.typeResolved().unresolved()); + } + + private static TestCaseSupplier.TestCase testCase(DataType strType, String str, Matcher matcher) { + return new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(new BytesRef(str), strType, "query")), + "EndsWithEvaluator[str=Attribute[channel=0], suffix=Attribute[channel=1]]", + DataType.BOOLEAN, + matcher + ); + } + + @Override + protected Expression build(Source source, List args) { + return new QueryStringFunction(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java index 81f391d637317..4dbdfd1e56854 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java @@ -113,6 +113,34 @@ public static Iterable parameters() { ) ) ); + suppliers.add(new TestCaseSupplier("Date Diff In Year - 1", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> { + ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01Z"); + ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z"); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("year"), DataType.KEYWORD, "unit"), + new TestCaseSupplier.TypedData(zdtStart2.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"), + new TestCaseSupplier.TypedData(zdtEnd2.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp") + ), + "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], " + "endTimestamp=Attribute[channel=2]]", + DataType.INTEGER, + equalTo(1) + ); + })); + suppliers.add(new TestCaseSupplier("Date Diff In Year - 0", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> { + ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01.001Z"); + ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z"); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("year"), DataType.KEYWORD, "unit"), + new TestCaseSupplier.TypedData(zdtStart2.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"), + new TestCaseSupplier.TypedData(zdtEnd2.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp") + ), + "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], " + "endTimestamp=Attribute[channel=2]]", + DataType.INTEGER, + equalTo(0) + ); + })); return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index eeb720084e635..2ed0093945837 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.ExchangeExec; import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec; +import org.elasticsearch.xpack.esql.plan.physical.FilterExec; import org.elasticsearch.xpack.esql.plan.physical.LimitExec; import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -373,6 +374,241 @@ public void testMultiCountAllWithFilter() { assertThat(plan.anyMatch(EsQueryExec.class::isInstance), is(true)); } + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"query_string":{"query":"last_name: Smith","fields":[]}}] + */ + public void testQueryStringFunction() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + var plan = plannerOptimizer.plan(""" + from test + | where qstr("last_name: Smith") + """, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + var expected = QueryBuilders.queryStringQuery("last_name: Smith"); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418],false] + * \_ProjectExec[[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#1414, gender{f}#1415, job{f}#1420, job.raw{f}#1421, langua + * ges{f}#1416, last_name{f}#1417, long_noidx{f}#1422, salary{f}#1418]] + * \_FieldExtractExec[_meta_field{f}#1419, emp_no{f}#1413, first_name{f}#] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}} + * ,{"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010"}}], + * "boost":1.0}}][_doc{f}#1423], limit[1000], sort[] estimatedRowSize[324] + */ + public void testQueryStringFunctionConjunctionWhereOperands() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") and emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 37, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(queryString).must(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n + * ame{f}#7, long_noidx{f}#12, salary{f}#8],false] + * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n + * ame{f}#7, long_noidx{f}#12, salary{f}#8]] + * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"should":[{"query_string":{"query":"last_name: Smith","fields":[]}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@2:37"}}], + * "boost":1.0}}][_doc{f}#13], limit[1000], sort[] estimatedRowSize[324] + */ + public void testQueryStringFunctionDisjunctionWhereClauses() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") or emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 36, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var expected = QueryBuilders.boolQuery().should(queryString).should(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16], + * false] + * \_ProjectExec[[!alias_integer, boolean{f}#4, byte{f}#5, constant_keyword-foo{f}#6, date{f}#7, double{f}#8, float{f}#9, half_ + * float{f}#10, integer{f}#12, ip{f}#13, keyword{f}#14, long{f}#15, scaled_float{f}#11, short{f}#17, text{f}#18, unsigned_long{f}#16] + * \_FieldExtractExec[!alias_integer, boolean{f}#4, byte{f}#5, constant_k..] + * \_EsQueryExec[test], indexMode[standard], query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}},{ + * "esql_single_value":{"field":"ip","next":{"terms":{"ip":["127.0.0.1/32"],"boost":1.0}}, + * "source":"cidr_match(ip, \"127.0.0.1/32\")@2:38"}}],"boost":1.0}}][_doc{f}#21], limit[1000], sort[] estimatedRowSize[354] + */ + public void testQueryStringFunctionWithFunctionsPushedToLucene() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") and cidr_match(ip, "127.0.0.1/32") + """; + var analyzer = makeAnalyzer("mapping-all-types.json", new EnrichResolution()); + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS, analyzer); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")"); + var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(queryString).must(terms); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + *LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n + * ame{f}#7, long_noidx{f}#12, salary{f}#8],false] + * \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, job{f}#10, job.raw{f}#11, languages{f}#6, last_n + * ame{f}#7, long_noidx{f}#12, salary{f}#8]] + * \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, gender{f}#5, job{f}#] + * \_LimitExec[1000[INTEGER]] + * \_FilterExec[LENGTH(first_name{f}#4) > 10[INTEGER]] + * \_FieldExtractExec[first_name{f}#4] + * \_EsQueryExec[test], indexMode[standard], + * query[{"query_string":{"query":"last_name: Smith","fields":[]}}][_doc{f}#13], limit[], sort[] estimatedRowSize[324] + */ + public void testQueryStringFunctionWithFunctionNotPushedDown() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") and length(first_name) > 10 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var firstLimit = as(plan, LimitExec.class); + var exchange = as(firstLimit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var secondLimit = as(field.child(), LimitExec.class); + var filter = as(secondLimit.child(), FilterExec.class); + var fieldExtract = as(filter.child(), FieldExtractExec.class); + var query = as(fieldExtract.child(), EsQueryExec.class); + + var expected = QueryBuilders.queryStringQuery("last_name: Smith"); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162],false] + * \_ProjectExec[[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#1158, gender{f}#1159, job{f}#1164, job.raw{f}#1165, langua + * ges{f}#1160, last_name{f}#1161, long_noidx{f}#1166, salary{f}#1162]] + * \_FieldExtractExec[_meta_field{f}#1163, emp_no{f}#1157, first_name{f}#] + * \_EsQueryExec[test], indexMode[standard], + * query[{"bool":{"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, + * {"esql_single_value":{"field":"emp_no","next":{"range":{"emp_no":{"gt":10010,"boost":1.0}}},"source":"emp_no > 10010@3:9"}}], + * "boost":1.0}}][_doc{f}#1167], limit[1000], sort[] estimatedRowSize[324] + */ + public void testQueryStringFunctionMultipleWhereClauses() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") + | where emp_no > 10010 + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + Source filterSource = new Source(3, 8, "emp_no > 10000"); + var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); + var expected = QueryBuilders.boolQuery().must(queryString).must(range); + assertThat(query.query().toString(), is(expected.toString())); + } + + /** + * Expecting + * LimitExec[1000[INTEGER]] + * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7],false] + * \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na + * me{f}#6, long_noidx{f}#11, salary{f}#7]] + * \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen] + * \_EsQueryExec[test], indexMode[standard], query[{"bool": + * {"must":[{"query_string":{"query":"last_name: Smith","fields":[]}}, + * {"query_string":{"query":"emp_no: [10010 TO *]","fields":[]}}],"boost":1.0}}] + */ + public void testQueryStringFunctionMultipleQstrClauses() { + assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()); + String queryText = """ + from test + | where qstr("last_name: Smith") and qstr("emp_no: [10010 TO *]") + """; + var plan = plannerOptimizer.plan(queryText, IS_SV_STATS); + + var limit = as(plan, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var field = as(project.child(), FieldExtractExec.class); + var query = as(field.child(), EsQueryExec.class); + assertThat(query.limit().fold(), is(1000)); + + var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith"); + var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]"); + var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight); + assertThat(query.query().toString(), is(expected.toString())); + } + /** * Expecting * LimitExec[1000[INTEGER]] diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index b1f38191ba7b3..6944f91042311 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -469,8 +469,6 @@ public long ramBytesUsed() { return 0; // Unknown } - @Override - public void close() {} }; } diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java index e728a74955f30..e52237f4d507e 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapper.java @@ -155,10 +155,6 @@ public long ramBytesUsed() { return 0; // Unknown } - @Override - public void close() { - // nothing to close - } }; } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java index 0c98c5b8bd5a7..00c60987f439d 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.fielddata.plain.FormattedSortedNumericDocValues; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.script.field.ToScriptFieldFactory; import org.elasticsearch.search.DocValueFormat; @@ -97,23 +98,7 @@ public void close() { @Override public FormattedDocValues getFormattedValues(DocValueFormat format) { - SortedNumericDocValues values = getLongValues(); - return new FormattedDocValues() { - @Override - public boolean advanceExact(int docId) throws IOException { - return values.advanceExact(docId); - } - - @Override - public int docValueCount() { - return values.docValueCount(); - } - - @Override - public Object nextValue() throws IOException { - return format.format(values.nextValue()); - } - }; + return new FormattedSortedNumericDocValues(getLongValues(), format); } static double convertUnsignedLongToDouble(long value) { diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index cdb25fbe995b2..f554a84048fde 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.NumberTypeOutOfRangeSpec; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.TimeSeriesParams; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.WholeNumberFieldMapperTests; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xcontent.XContentBuilder; @@ -268,11 +269,11 @@ public void testDimensionMultiValuedFieldTSDB() throws IOException { b.field("time_series_dimension", true); }), IndexMode.TIME_SERIES); - Exception e = expectThrows( - DocumentParsingException.class, - () -> mapper.parse(source(b -> b.array("field", randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()))) - ); - assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); + ParsedDocument doc = mapper.parse(source(null, b -> { + b.array("field", randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); + b.field("@timestamp", Instant.now()); + }, TimeSeriesRoutingHashFieldMapper.encode(randomInt()))); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); } public void testDimensionMultiValuedFieldNonTSDB() throws IOException { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java index afd17b803cdcb..c86b3e710a736 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentNodeService.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchPhaseExecutionException; @@ -775,7 +776,11 @@ private void updateStoredState(String deploymentId, RoutingInfoUpdate update, Ac } private void handleLoadFailure(TrainedModelDeploymentTask task, Exception ex, ActionListener retryListener) { - logger.error(() -> "[" + task.getDeploymentId() + "] model [" + task.getParams().getModelId() + "] failed to load", ex); + if (ex instanceof ElasticsearchException esEx && esEx.status().getStatus() < 500) { + logger.warn(() -> "[" + task.getDeploymentId() + "] model [" + task.getParams().getModelId() + "] failed to load", ex); + } else { + logger.error(() -> "[" + task.getDeploymentId() + "] model [" + task.getParams().getModelId() + "] failed to load", ex); + } if (task.isStopped()) { logger.debug( () -> format( diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index 89ff28249aabb..a4413d266181d 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -28,6 +28,11 @@ template: type: constant_keyword value: metrics dynamic_templates: + - ecs_ip: + mapping: + type: ip + path_match: [ "ip", "*.ip", "*_ip" ] + match_mapping_type: string - all_strings_to_keywords: mapping: ignore_above: 1024 diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml index a6591d6c32210..1823dfab7e716 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_metrics_tests.yml @@ -123,11 +123,10 @@ setup: start_time: 2024-07-01T13:03:08.138Z mappings: dynamic_templates: - - ip_fields: + - no_ip_fields: mapping: - type: ip + type: keyword match_mapping_type: string - path_match: "*.ip" - do: bulk: index: metrics-generic.otel-default @@ -145,5 +144,37 @@ setup: indices.get_mapping: index: $idx0name expand_wildcards: hidden - - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' } + - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'keyword' } - match: { .$idx0name.mappings.properties.attributes.properties.foo.type: "keyword" } +--- +IP dimensions: + - requires: + cluster_features: ["routing.multi_value_routing_path"] + reason: support for multi-value dimensions + - do: + bulk: + index: metrics-generic.otel-default + refresh: true + body: + - create: {"dynamic_templates":{"metrics.foo.bar":"counter_long"}} + - "@timestamp": 2024-07-18T14:48:33.467654000Z + resource: + attributes: + host.ip: [ "127.0.0.1", "0.0.0.0" ] + attributes: + philip: [ a, b, c ] + metrics: + foo.bar: 42 + - is_false: errors + + - do: + indices.get_data_stream: + name: metrics-generic.otel-default + - set: { data_streams.0.indices.0.index_name: idx0name } + + - do: + indices.get_mapping: + index: $idx0name + expand_wildcards: hidden + - match: { .$idx0name.mappings.properties.resource.properties.attributes.properties.host\.ip.type: 'ip' } + - match: { .$idx0name.mappings.properties.attributes.properties.philip.type: "keyword" } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java index f4c3b77af3abe..28c75987248ee 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/NativeRealmIntegTestCase.java @@ -36,7 +36,7 @@ public abstract class NativeRealmIntegTestCase extends SecurityIntegTestCase { @Before public void ensureNativeStoresStarted() throws Exception { - assertSecurityIndexActive(); + createSecurityIndexWithWaitForActiveShards(); if (shouldSetReservedUserPasswords()) { setupReservedPasswords(); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index f164189c6f047..70dcfbaa315cf 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -191,7 +191,7 @@ protected boolean addMockHttpTransport() { @Before public void waitForSecurityIndexWritable() throws Exception { - assertSecurityIndexActive(); + createSecurityIndexWithWaitForActiveShards(); } @After diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index 66ea1235800ed..fef1a98ca67e9 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -783,7 +783,7 @@ public void testAuthenticateWithWrongToken() throws Exception { @Before public void waitForSecurityIndexWritable() throws Exception { - assertSecurityIndexActive(); + createSecurityIndexWithWaitForActiveShards(); } @After diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java index 1b62c79236a9c..eb7c5e5276c15 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java @@ -31,7 +31,7 @@ public class SecurityScrollTests extends SecurityIntegTestCase { public void testScrollIsPerUser() throws Exception { - assertSecurityIndexActive(); + createSecurityIndexWithWaitForActiveShards(); new PutRoleRequestBuilder(client()).name("scrollable") .addIndices(new String[] { randomAlphaOfLengthBetween(4, 12) }, new String[] { "read" }, null, null, null, randomBoolean()) .get(); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java index 4bbffc139cbfd..32337f0d66896 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java @@ -46,7 +46,6 @@ public class SecurityIndexManagerIntegTests extends SecurityIntegTestCase { public void testConcurrentOperationsTryingToCreateSecurityIndexAndAlias() throws Exception { - assertSecurityIndexActive(); final int processors = Runtime.getRuntime().availableProcessors(); final int numThreads = Math.min(50, scaledRandomIntBetween((processors + 1) / 2, 4 * processors)); // up to 50 threads final int maxNumRequests = 50 / numThreads; // bound to a maximum of 50 requests @@ -111,7 +110,7 @@ public void testOnIndexAvailableForSearchIndexCompletesWithinTimeout() throws Ex // pick longer wait than in the assertBusy that waits for below to ensure index has had enough time to initialize securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueSeconds(40)); - createSecurityIndex(); + createSecurityIndexWithWaitForActiveShards(); assertBusy( () -> assertThat(securityIndexManager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), is(true)), @@ -126,7 +125,7 @@ public void testOnIndexAvailableForSearchIndexCompletesWithinTimeout() throws Ex @SuppressWarnings("unchecked") public void testOnIndexAvailableForSearchIndexAlreadyAvailable() throws Exception { - createSecurityIndex(); + createSecurityIndexWithWaitForActiveShards(); final SecurityIndexManager securityIndexManager = internalCluster().getInstances(NativePrivilegeStore.class) .iterator() diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index ddcef0e24727d..3a39d54567726 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.internal.Client; @@ -381,6 +382,12 @@ protected Function getClientWrapper() { return client -> (client instanceof NodeClient) ? client.filterWithHeader(headers) : client; } + /** + * Waits for security index to become available. Note that you must ensure index creation was triggered before calling this method, + * by calling one of the resource creation APIs (e.g., creating a user). + * If you use {@link #createSecurityIndexWithWaitForActiveShards()} to create the index it's not necessary to call + * {@link #assertSecurityIndexActive} since the create method ensures the index is active. + */ public void assertSecurityIndexActive() throws Exception { assertSecurityIndexActive(cluster()); } @@ -391,14 +398,10 @@ public void assertSecurityIndexActive(TestCluster testCluster) throws Exception ClusterState clusterState = client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState(); assertFalse(clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)); Index securityIndex = resolveSecurityIndex(clusterState.metadata()); - // TODO this is a bug -- since we are not tripping assertions here, this will complete successfully even if the security - // index does not exist - if (securityIndex != null) { - IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(securityIndex); - if (indexRoutingTable != null) { - assertTrue(indexRoutingTable.allPrimaryShardsActive()); - } - } + assertNotNull(securityIndex); + IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(securityIndex); + assertNotNull(indexRoutingTable); + assertTrue(indexRoutingTable.allPrimaryShardsActive()); }, 30L, TimeUnit.SECONDS); } } @@ -424,7 +427,7 @@ protected void deleteSecurityIndex() { } } - protected void createSecurityIndex() { + protected void createSecurityIndexWithWaitForActiveShards() { final Client client = client().filterWithHeader( Collections.singletonMap( "Authorization", @@ -434,7 +437,8 @@ protected void createSecurityIndex() { ) ) ); - CreateIndexRequest createIndexRequest = new CreateIndexRequest(SECURITY_MAIN_ALIAS); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(SECURITY_MAIN_ALIAS).waitForActiveShards(ActiveShardCount.ALL) + .masterNodeTimeout(TEST_REQUEST_TIMEOUT); client.admin().indices().create(createIndexRequest).actionGet(); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LeafShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LeafShapeFieldData.java index f3a3b4d2f3f27..8be44242886c9 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LeafShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LeafShapeFieldData.java @@ -38,9 +38,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() {} - @Override public T getShapeValues() { return emptyValues; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianPointDVLeafFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianPointDVLeafFieldData.java index 9cbbe04f56cd5..b8bd5e82fef3b 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianPointDVLeafFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianPointDVLeafFieldData.java @@ -34,11 +34,6 @@ public long ramBytesUsed() { return 0; // not exposed by lucene } - @Override - public void close() { - // noop - } - @Override public SortedNumericDocValues getSortedNumericDocValues() { try { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianShapeDVAtomicShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianShapeDVAtomicShapeFieldData.java index 34e6c569a4e68..42eb1925601cd 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianShapeDVAtomicShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/CartesianShapeDVAtomicShapeFieldData.java @@ -37,11 +37,6 @@ public long ramBytesUsed() { return 0; // not exposed by lucene } - @Override - public void close() { - // noop - } - @Override public CartesianShapeValues getShapeValues() { try { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/GeoShapeScriptFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/GeoShapeScriptFieldData.java index 4790924412cd7..bf4202466eb41 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/GeoShapeScriptFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/GeoShapeScriptFieldData.java @@ -70,10 +70,6 @@ public long ramBytesUsed() { return 0; } - @Override - public void close() { - - } }; } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java index 8319da1306be6..a1aedb07825f3 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/LatLonShapeDVAtomicShapeFieldData.java @@ -33,11 +33,6 @@ public long ramBytesUsed() { return 0; // not exposed by lucene } - @Override - public void close() { - // noop - } - @Override public GeoShapeValues getShapeValues() { try { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/CartesianPointValuesSource.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/CartesianPointValuesSource.java index 5d916b6c5cead..68d5e3bd04421 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/CartesianPointValuesSource.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/support/CartesianPointValuesSource.java @@ -9,18 +9,14 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.common.Rounding; import org.elasticsearch.index.fielddata.DocValueBits; import org.elasticsearch.index.fielddata.MultiPointValues; -import org.elasticsearch.index.fielddata.PointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.search.aggregations.AggregationErrors; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.xcontent.ToXContentFragment; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.spatial.common.CartesianPoint; import org.elasticsearch.xpack.spatial.index.fielddata.IndexCartesianPointFieldData; @@ -73,33 +69,6 @@ public CartesianPoint nextValue() throws IOException { return point.resetFromEncoded(numericValues.nextValue()); } - @Override - protected PointValues getPointValues() { - final NumericDocValues singleton = DocValues.unwrapSingleton(numericValues); - return singleton != null ? new CartesianPointValues(singleton) : null; - } - } - - public static final class CartesianPointValues extends PointValues { - - private final CartesianPoint point = new CartesianPoint(); - - CartesianPointValues(NumericDocValues values) { - super(values); - } - - @Override - public CartesianPoint pointValue() throws IOException { - return point.resetFromEncoded(values.longValue()); - } - } - - public static final class CartesianPointValue implements ToXContentFragment { - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return null; - } } /** diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/10_forbidden.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/10_forbidden.yml new file mode 100644 index 0000000000000..545b9ef1e0285 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/10_forbidden.yml @@ -0,0 +1,31 @@ +--- +"Test basic response with invalid credentials": + + - skip: + features: headers + + - do: + headers: + Authorization: "Basic dGVzdF91c2VyOndyb25nLXBhc3N3b3Jk" # invalid credentials + info: {} + catch: unauthorized + + - match: + error.root_cause.0.type: security_exception + +--- +"Test bulk response with invalid credentials": + + - skip: + features: headers + + - do: + headers: + Authorization: "Basic dGVzdF91c2VyOndyb25nLXBhc3N3b3Jk" # invalid credentials + bulk: + body: | + {"index": {}} + {} + catch: unauthorized + - match: + error.root_cause.0.type: security_exception diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/13_index_datemath.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/13_index_datemath.yml index db1dd72553b3f..eee0e273974bd 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/13_index_datemath.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/13_index_datemath.yml @@ -1,5 +1,7 @@ --- setup: + - requires: + test_runner_features: allowed_warnings - skip: features: headers @@ -54,6 +56,8 @@ teardown: } - do: + allowed_warnings: + - 'Date format [YYYY.MM] contains week-date field specifiers that are changing in JDK 23' headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user bulk: body: @@ -89,6 +93,8 @@ teardown: } - do: + allowed_warnings: + - 'Date format [YYYY.MM] contains week-date field specifiers that are changing in JDK 23' headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user bulk: body: @@ -115,6 +121,8 @@ teardown: --- "Test bulk indexing with datemath when only some are allowed": - do: + allowed_warnings: + - 'Date format [YYYY] contains week-date field specifiers that are changing in JDK 23' headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user bulk: body: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml index 1d370082c8e48..e1b297f1b5d78 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml @@ -90,3 +90,53 @@ setup: - match: {hits.total: 1 } - length: {hits.hits: 1 } - match: {hits.hits.0._id: "1" } + +--- +"Failed to snapshot indices with synthetic source": + - skip: + features: ["allowed_warnings"] + + - do: + indices.create: + index: test_synthetic + body: + mappings: + _source: + mode: synthetic + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + snapshot.create: + repository: test_repo_restore_1 + snapshot: test_snapshot_2 + wait_for_completion: true + body: | + { "indices": "test_synthetic" } + + - match: { snapshot.snapshot: test_snapshot_2 } + - match: { snapshot.state : PARTIAL } + - match: { snapshot.shards.successful: 0 } + - match: { snapshot.shards.failed : 1 } + - match: { snapshot.failures.0.index: "test_synthetic" } + - match: { snapshot.failures.0.reason : "IllegalStateException[Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source]" } + - is_true: snapshot.version + - gt: { snapshot.version_id: 0} + + - do: + snapshot.create: + repository: test_repo_restore_1 + snapshot: test_snapshot_3 + wait_for_completion: true + body: | + { "indices": "test_*" } + + - match: { snapshot.snapshot: test_snapshot_3 } + - match: { snapshot.state : PARTIAL } + - match: { snapshot.shards.successful: 1 } + - match: { snapshot.shards.failed : 1 } + - match: { snapshot.failures.0.index: "test_synthetic" } + - match: { snapshot.failures.0.reason: "IllegalStateException[Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source]" } + - is_true: snapshot.version + - gt: { snapshot.version_id: 0} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java index 9212780d11fd3..5e6363b993ce4 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java @@ -16,14 +16,12 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -69,42 +67,35 @@ protected void masterOperation( final boolean manuallyStopped = request.getCommand() == WatcherServiceRequest.Command.STOP; final String source = manuallyStopped ? "update_watcher_manually_stopped" : "update_watcher_manually_started"; - // TODO: make WatcherServiceRequest a real AckedRequest so that we have both a configurable timeout and master node timeout like - // we do elsewhere - submitUnbatchedTask(source, new AckedClusterStateUpdateTask(new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return AcknowledgedRequest.DEFAULT_ACK_TIMEOUT; - } - - @Override - public TimeValue masterNodeTimeout() { - return request.masterNodeTimeout(); - } - }, listener) { - @Override - public ClusterState execute(ClusterState clusterState) { - XPackPlugin.checkReadyForXPackCustomMetadata(clusterState); + // TODO: make WatcherServiceRequest a real AcknowledgedRequest so that we have both a configurable timeout and master node timeout + // like we do elsewhere + submitUnbatchedTask( + source, + new AckedClusterStateUpdateTask(request.masterNodeTimeout(), AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, listener) { + @Override + public ClusterState execute(ClusterState clusterState) { + XPackPlugin.checkReadyForXPackCustomMetadata(clusterState); - WatcherMetadata newWatcherMetadata = new WatcherMetadata(manuallyStopped); - WatcherMetadata currentMetadata = clusterState.metadata().custom(WatcherMetadata.TYPE); + WatcherMetadata newWatcherMetadata = new WatcherMetadata(manuallyStopped); + WatcherMetadata currentMetadata = clusterState.metadata().custom(WatcherMetadata.TYPE); - // adhere to the contract of returning the original state if nothing has changed - if (newWatcherMetadata.equals(currentMetadata)) { - return clusterState; - } else { - ClusterState.Builder builder = new ClusterState.Builder(clusterState); - builder.metadata(Metadata.builder(clusterState.getMetadata()).putCustom(WatcherMetadata.TYPE, newWatcherMetadata)); - return builder.build(); + // adhere to the contract of returning the original state if nothing has changed + if (newWatcherMetadata.equals(currentMetadata)) { + return clusterState; + } else { + ClusterState.Builder builder = new ClusterState.Builder(clusterState); + builder.metadata(Metadata.builder(clusterState.getMetadata()).putCustom(WatcherMetadata.TYPE, newWatcherMetadata)); + return builder.build(); + } } - } - @Override - public void onFailure(Exception e) { - logger.error(() -> format("could not update watcher stopped status to [%s], source [%s]", manuallyStopped, source), e); - listener.onFailure(e); + @Override + public void onFailure(Exception e) { + logger.error(() -> format("could not update watcher stopped status to [%s], source [%s]", manuallyStopped, source), e); + listener.onFailure(e); + } } - }); + ); } @Override diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java index 17363d58545c2..87c7e344336ad 100644 --- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java +++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java @@ -181,7 +181,7 @@ protected Settings buildRealmSettings( @Before public void setupRoleMappings() throws Exception { - assertSecurityIndexActive(); + createSecurityIndexWithWaitForActiveShards(); List content = getRoleMappingContent(RoleMappingEntry::nativeContent); if (content.isEmpty()) {