diff --git a/.ci/dockerOnLinuxExclusions b/.ci/dockerOnLinuxExclusions index c150cca590f7d..715ed86188dd5 100644 --- a/.ci/dockerOnLinuxExclusions +++ b/.ci/dockerOnLinuxExclusions @@ -15,6 +15,7 @@ sles-15.2 sles-15.3 sles-15.4 sles-15.5 +sles-15.6 # These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded # for PR checks diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java new file mode 100644 index 0000000000000..13a222b1ed35e --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.benchmark.indices.resolution; + +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.SystemIndices; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +@State(Scope.Benchmark) +@Fork(3) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@SuppressWarnings("unused") // invoked by benchmarking framework +public class IndexNameExpressionResolverBenchmark { + + private static final String DATA_STREAM_PREFIX = "my-ds-"; + private static final String INDEX_PREFIX = "my-index-"; + + @Param( + { + // # data streams | # indices + " 1000| 100", + " 5000| 500", + " 10000| 1000" } + ) + public String resourceMix = "100|10"; + + @Setup + public void setUp() { + final String[] params = resourceMix.split("\\|"); + + int numDataStreams = toInt(params[0]); + int numIndices = toInt(params[1]); + + Metadata.Builder mb = Metadata.builder(); + String[] indices = new String[numIndices + numDataStreams * (numIndices + 1)]; + int position = 0; + for (int i = 1; i <= numIndices; i++) { + String indexName = INDEX_PREFIX + i; + createIndexMetadata(indexName, mb); + indices[position++] = indexName; + } + + for (int i = 1; i <= numDataStreams; i++) { + String dataStreamName = DATA_STREAM_PREFIX + i; + List backingIndices = new ArrayList<>(); + for (int j = 1; j <= numIndices; j++) { + String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, j); + backingIndices.add(createIndexMetadata(backingIndexName, mb).getIndex()); + indices[position++] = backingIndexName; + } + indices[position++] = dataStreamName; + mb.put(DataStream.builder(dataStreamName, backingIndices).build()); + } + int mid = indices.length / 2; + clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build(); + resolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), new SystemIndices(List.of())); + indexListRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), indices); + starRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), "*"); + String[] mixed = indices.clone(); + mixed[mid] = "my-*"; + mixedRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), mixed); + } + + private IndexMetadata createIndexMetadata(String indexName, Metadata.Builder mb) { + IndexMetadata indexMetadata = IndexMetadata.builder(indexName) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + mb.put(indexMetadata, false); + return indexMetadata; + } + + private IndexNameExpressionResolver resolver; + private ClusterState clusterState; + private Request starRequest; + private Request indexListRequest; + private Request mixedRequest; + + @Benchmark + public String[] resolveResourcesListToConcreteIndices() { + return resolver.concreteIndexNames(clusterState, indexListRequest); + } + + @Benchmark + public String[] resolveAllStarToConcreteIndices() { + return resolver.concreteIndexNames(clusterState, starRequest); + } + + @Benchmark + public String[] resolveMixedConcreteIndices() { + return resolver.concreteIndexNames(clusterState, mixedRequest); + } + + private int toInt(String v) { + return Integer.parseInt(v.trim()); + } + + record Request(IndicesOptions indicesOptions, String... indices) implements IndicesRequest { + + } +} diff --git a/docs/changelog/114484.yaml b/docs/changelog/114484.yaml new file mode 100644 index 0000000000000..48f54ad0218bb --- /dev/null +++ b/docs/changelog/114484.yaml @@ -0,0 +1,6 @@ +pr: 114484 +summary: Add `docvalue_fields` Support for `dense_vector` Fields +area: Search +type: enhancement +issues: + - 108470 diff --git a/docs/changelog/114964.yaml b/docs/changelog/114964.yaml new file mode 100644 index 0000000000000..8274aeb76a937 --- /dev/null +++ b/docs/changelog/114964.yaml @@ -0,0 +1,6 @@ +pr: 114964 +summary: Add a `monitor_stats` privilege and allow that privilege for remote cluster + privileges +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/116325.yaml b/docs/changelog/116325.yaml new file mode 100644 index 0000000000000..b8cd16dc85773 --- /dev/null +++ b/docs/changelog/116325.yaml @@ -0,0 +1,5 @@ +pr: 116325 +summary: Adjust analyze limit exception to be a `bad_request` +area: Analysis +type: bug +issues: [] diff --git a/docs/changelog/116382.yaml b/docs/changelog/116382.yaml new file mode 100644 index 0000000000000..c941fb6eaa1e4 --- /dev/null +++ b/docs/changelog/116382.yaml @@ -0,0 +1,5 @@ +pr: 116382 +summary: Validate missing shards after the coordinator rewrite +area: Search +type: bug +issues: [] diff --git a/docs/changelog/116478.yaml b/docs/changelog/116478.yaml new file mode 100644 index 0000000000000..ec50799eb2019 --- /dev/null +++ b/docs/changelog/116478.yaml @@ -0,0 +1,5 @@ +pr: 116478 +summary: Semantic text simple partial update +area: Search +type: bug +issues: [] diff --git a/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc index 658470c8d5a4e..d5bd868258081 100644 --- a/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc @@ -127,10 +127,11 @@ And the following may be the response: ==== Percentiles_bucket implementation -The Percentile Bucket returns the nearest input data point that is not greater than the requested percentile; it does not -interpolate between data points. - The percentiles are calculated exactly and is not an approximation (unlike the Percentiles Metric). This means the implementation maintains an in-memory, sorted list of your data to compute the percentiles, before discarding the data. You may run into memory pressure issues if you attempt to calculate percentiles over many millions of data-points in a single `percentiles_bucket`. + +The Percentile Bucket returns the nearest input data point to the requested percentile, rounding indices toward +positive infinity; it does not interpolate between data points. For example, if there are eight data points and +you request the `50%th` percentile, it will return the `4th` item because `ROUND_UP(.50 * (8-1))` is `4`. diff --git a/docs/reference/esql/esql-kibana.asciidoc b/docs/reference/esql/esql-kibana.asciidoc index 9850e012fc049..85969e19957af 100644 --- a/docs/reference/esql/esql-kibana.asciidoc +++ b/docs/reference/esql/esql-kibana.asciidoc @@ -9,9 +9,9 @@ You can use {esql} in {kib} to query and aggregate your data, create visualizations, and set up alerts. This guide shows you how to use {esql} in Kibana. To follow along with the -queries, load the "Sample web logs" sample data set by clicking *Try sample -data* from the {kib} Home, selecting *Other sample data sets*, and clicking *Add -data* on the *Sample web logs* card. +queries, load the "Sample web logs" sample data set by selecting **Sample Data** +from the **Integrations** page in {kib}, selecting *Other sample data sets*, +and clicking *Add data* on the *Sample web logs* card. [discrete] [[esql-kibana-enable]] @@ -30,9 +30,7 @@ However, users will be able to access existing {esql} artifacts like saved searc // tag::esql-mode[] To get started with {esql} in Discover, open the main menu and select -*Discover*. Next, from the Data views menu, select *Language: ES|QL*. - -image::images/esql/esql-data-view-menu.png[align="center",width=33%] +*Discover*. Next, select *Try ES|QL* from the application menu bar. // end::esql-mode[] [discrete] @@ -54,8 +52,9 @@ A source command can be followed by one or more <>. In this query, the processing command is <>. `LIMIT` limits the number of rows that are retrieved. -TIP: Click the help icon (image:images/esql/esql-icon-help.svg[Static,20]) to open the -in-product reference documentation for all commands and functions. +TIP: Click the **ES|QL help** button to open the +in-product reference documentation for all commands and functions or to get +recommended queries that will help you get started. // tag::autocomplete[] To make it easier to write queries, auto-complete offers suggestions with @@ -76,7 +75,7 @@ FROM kibana_sample_data_logs | LIMIT 10 ==== [discrete] -==== Expand the query bar +==== Make your query readable For readability, you can put each processing command on a new line. The following query is identical to the previous one: @@ -87,15 +86,12 @@ FROM kibana_sample_data_logs | LIMIT 10 ---- -// tag::compact[] -To make it easier to write multi-line queries, click the double-headed arrow -button (image:images/esql/esql-icon-expand-query-bar.svg[]) to expand the query -bar: +You can do that using the **Add line breaks on pipes** button from the query editor's footer. -image::images/esql/esql-expanded-query-bar.png[align="center"] +image::https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/bltd5554518309e10f6/672d153cfeb8f9d479ebcc6e/esql-line-breakdown.gif[Automatic line breaks for ES|QL queries] -To return to a compact query bar, click the minimize editor button -(image:images/esql/esql-icon-minimize-query-bar.svg[]). +// tag::compact[] +You can adjust the editor's height by dragging its bottom border to your liking. // end::compact[] [discrete] @@ -110,9 +106,7 @@ detailed warning, expand the query bar, and click *warnings*. ==== Query history You can reuse your recent {esql} queries in the query bar. -In the query bar click *Show recent queries*: - -image::images/esql/esql-discover-show-recent-query.png[align="center",size="50%"] +In the query bar click *Show recent queries*. You can then scroll through your recent queries: @@ -220,8 +214,9 @@ FROM kibana_sample_data_logs === Analyze and visualize data Between the query bar and the results table, Discover shows a date histogram -visualization. If the indices you're querying do not contain a `@timestamp` -field, the histogram is not shown. +visualization. By default, if the indices you're querying do not contain a `@timestamp` +field, the histogram is not shown. But you can use a custom time field with the `?_tstart` +and `?_tend` parameters to enable it. The visualization adapts to the query. A query's nature determines the type of visualization. For example, this query aggregates the total number of bytes per @@ -250,7 +245,7 @@ save button (image:images/esql/esql-icon-save-visualization.svg[]). Once saved to a dashboard, you'll be taken to the Dashboards page. You can continue to make changes to the visualization. Click the options button in the top-right (image:images/esql/esql-icon-options.svg[]) and -select *Edit ESQL visualization* to open the in-line editor: +select *Edit ES|QL visualization* to open the in-line editor: image::images/esql/esql-kibana-edit-on-dashboard.png[align="center",width=66%] diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc index 83614b0d99024..e884c01dd3509 100644 --- a/docs/reference/how-to/knn-search.asciidoc +++ b/docs/reference/how-to/knn-search.asciidoc @@ -72,15 +72,13 @@ least enough RAM to hold the vector data and index structures. To check the size of the vector data, you can use the <> API. Here are estimates for different element types and quantization levels: -+ --- -`element_type: float`: `num_vectors * num_dimensions * 4` -`element_type: float` with `quantization: int8`: `num_vectors * (num_dimensions + 4)` -`element_type: float` with `quantization: int4`: `num_vectors * (num_dimensions/2 + 4)` -`element_type: float` with `quantization: bbq`: `num_vectors * (num_dimensions/8 + 12)` -`element_type: byte`: `num_vectors * num_dimensions` -`element_type: bit`: `num_vectors * (num_dimensions/8)` --- + +* `element_type: float`: `num_vectors * num_dimensions * 4` +* `element_type: float` with `quantization: int8`: `num_vectors * (num_dimensions + 4)` +* `element_type: float` with `quantization: int4`: `num_vectors * (num_dimensions/2 + 4)` +* `element_type: float` with `quantization: bbq`: `num_vectors * (num_dimensions/8 + 12)` +* `element_type: byte`: `num_vectors * num_dimensions` +* `element_type: bit`: `num_vectors * (num_dimensions/8)` If utilizing HNSW, the graph must also be in memory, to estimate the required bytes use `num_vectors * 4 * HNSW.m`. The default value for `HNSW.m` is 16, so by default `num_vectors * 4 * 16`. diff --git a/docs/reference/images/esql/esql-dashboard-panel.png b/docs/reference/images/esql/esql-dashboard-panel.png index d621d1170edcf..61b44f7c9f857 100644 Binary files a/docs/reference/images/esql/esql-dashboard-panel.png and b/docs/reference/images/esql/esql-dashboard-panel.png differ diff --git a/docs/reference/images/esql/esql-discover-query-history.png b/docs/reference/images/esql/esql-discover-query-history.png index da31e4a6acce4..ff1d2ffa8b280 100644 Binary files a/docs/reference/images/esql/esql-discover-query-history.png and b/docs/reference/images/esql/esql-discover-query-history.png differ diff --git a/docs/reference/images/esql/esql-discover-show-recent-query.png b/docs/reference/images/esql/esql-discover-show-recent-query.png deleted file mode 100644 index 13c8df9965ea3..0000000000000 Binary files a/docs/reference/images/esql/esql-discover-show-recent-query.png and /dev/null differ diff --git a/docs/reference/images/esql/esql-kibana-auto-complete.png b/docs/reference/images/esql/esql-kibana-auto-complete.png index d50d6b133442f..155df2447dd6c 100644 Binary files a/docs/reference/images/esql/esql-kibana-auto-complete.png and b/docs/reference/images/esql/esql-kibana-auto-complete.png differ diff --git a/docs/reference/images/esql/esql-kibana-bar-chart.png b/docs/reference/images/esql/esql-kibana-bar-chart.png index a760d3d69920e..b74b33710d908 100644 Binary files a/docs/reference/images/esql/esql-kibana-bar-chart.png and b/docs/reference/images/esql/esql-kibana-bar-chart.png differ diff --git a/docs/reference/images/esql/esql-kibana-create-rule.png b/docs/reference/images/esql/esql-kibana-create-rule.png index c9fb14b0d2ee9..a763c6f366df0 100644 Binary files a/docs/reference/images/esql/esql-kibana-create-rule.png and b/docs/reference/images/esql/esql-kibana-create-rule.png differ diff --git a/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png b/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png index 14f6be81af7df..348b77150ed87 100644 Binary files a/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png and b/docs/reference/images/esql/esql-kibana-edit-on-dashboard.png differ diff --git a/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png b/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png index 95a997ca2ac30..f2a0779be348a 100644 Binary files a/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png and b/docs/reference/images/esql/esql-kibana-enrich-autocomplete.png differ diff --git a/docs/reference/images/esql/esql-kibana-in-line-editor.png b/docs/reference/images/esql/esql-kibana-in-line-editor.png index 7b7a11e532226..85631896e833f 100644 Binary files a/docs/reference/images/esql/esql-kibana-in-line-editor.png and b/docs/reference/images/esql/esql-kibana-in-line-editor.png differ diff --git a/docs/reference/rest-api/security/bulk-create-roles.asciidoc b/docs/reference/rest-api/security/bulk-create-roles.asciidoc index a1fe998c08146..a198f49383907 100644 --- a/docs/reference/rest-api/security/bulk-create-roles.asciidoc +++ b/docs/reference/rest-api/security/bulk-create-roles.asciidoc @@ -327,7 +327,7 @@ The result would then have the `errors` field set to `true` and hold the error f "details": { "my_admin_role": { <4> "type": "action_request_validation_exception", - "reason": "Validation Failed: 1: unknown cluster privilege [bad_cluster_privilege]. a privilege must be either one of the predefined cluster privilege names [manage_own_api_key,manage_data_stream_global_retention,monitor_data_stream_global_retention,none,cancel_task,cross_cluster_replication,cross_cluster_search,delegate_pki,grant_api_key,manage_autoscaling,manage_index_templates,manage_logstash_pipelines,manage_oidc,manage_saml,manage_search_application,manage_search_query_rules,manage_search_synonyms,manage_service_account,manage_token,manage_user_profile,monitor_connector,monitor_enrich,monitor_inference,monitor_ml,monitor_rollup,monitor_snapshot,monitor_text_structure,monitor_watcher,post_behavioral_analytics_event,read_ccr,read_connector_secrets,read_fleet_secrets,read_ilm,read_pipeline,read_security,read_slm,transport_client,write_connector_secrets,write_fleet_secrets,create_snapshot,manage_behavioral_analytics,manage_ccr,manage_connector,manage_enrich,manage_ilm,manage_inference,manage_ml,manage_rollup,manage_slm,manage_watcher,monitor_data_frame_transforms,monitor_transform,manage_api_key,manage_ingest_pipelines,manage_pipeline,manage_data_frame_transforms,manage_transform,manage_security,monitor,manage,all] or a pattern over one of the available cluster actions;" + "reason": "Validation Failed: 1: unknown cluster privilege [bad_cluster_privilege]. a privilege must be either one of the predefined cluster privilege names [manage_own_api_key,manage_data_stream_global_retention,monitor_data_stream_global_retention,none,cancel_task,cross_cluster_replication,cross_cluster_search,delegate_pki,grant_api_key,manage_autoscaling,manage_index_templates,manage_logstash_pipelines,manage_oidc,manage_saml,manage_search_application,manage_search_query_rules,manage_search_synonyms,manage_service_account,manage_token,manage_user_profile,monitor_connector,monitor_enrich,monitor_inference,monitor_ml,monitor_rollup,monitor_snapshot,monitor_stats,monitor_text_structure,monitor_watcher,post_behavioral_analytics_event,read_ccr,read_connector_secrets,read_fleet_secrets,read_ilm,read_pipeline,read_security,read_slm,transport_client,write_connector_secrets,write_fleet_secrets,create_snapshot,manage_behavioral_analytics,manage_ccr,manage_connector,manage_enrich,manage_ilm,manage_inference,manage_ml,manage_rollup,manage_slm,manage_watcher,monitor_data_frame_transforms,monitor_transform,manage_api_key,manage_ingest_pipelines,manage_pipeline,manage_data_frame_transforms,manage_transform,manage_security,monitor,manage,all] or a pattern over one of the available cluster actions;" } } } diff --git a/docs/reference/rest-api/security/get-builtin-privileges.asciidoc b/docs/reference/rest-api/security/get-builtin-privileges.asciidoc index 8435f5539ab9d..7f3d75b926780 100644 --- a/docs/reference/rest-api/security/get-builtin-privileges.asciidoc +++ b/docs/reference/rest-api/security/get-builtin-privileges.asciidoc @@ -111,6 +111,7 @@ A successful call returns an object with "cluster", "index", and "remote_cluster "monitor_ml", "monitor_rollup", "monitor_snapshot", + "monitor_stats", "monitor_text_structure", "monitor_transform", "monitor_watcher", @@ -152,7 +153,8 @@ A successful call returns an object with "cluster", "index", and "remote_cluster "write" ], "remote_cluster" : [ - "monitor_enrich" + "monitor_enrich", + "monitor_stats" ] } -------------------------------------------------- diff --git a/docs/reference/search/search-your-data/retrievers-examples.asciidoc b/docs/reference/search/search-your-data/retrievers-examples.asciidoc new file mode 100644 index 0000000000000..8cd1a4bf5ce98 --- /dev/null +++ b/docs/reference/search/search-your-data/retrievers-examples.asciidoc @@ -0,0 +1,428 @@ +[[retrievers-examples]] +=== Retrievers examples + +Learn how to combine different retrievers in these hands-on examples. +To demonstrate the full functionality of retrievers, these examples require access to a <> set up using the <>. + +[discrete] +[[retrievers-examples-setup]] +==== Add example data + +To begin with, we'll set up the necessary services and have them in place for later use. + +[source,js] +---- +// Setup rerank task stored as `my-rerank-model` +PUT _inference/rerank/my-rerank-model +{ + "service": "cohere", + "service_settings": { + "model_id": "rerank-english-v3.0", + "api_key": "{{COHERE_API_KEY}}" + } +} +---- +//NOTCONSOLE + +Now that we have our reranking service in place, lets create the `retrievers_example` index, and add some documents to it. +[source,js] +---- +PUT retrievers_example +{ + "mappings": { + "properties": { + "vector": { + "type": "dense_vector", + "dims": 3, + "similarity": "l2_norm", + "index": true + }, + "text": { + "type": "text" + }, + "year": { + "type": "integer" + }, + "topic": { + "type": "keyword" + } + } + } +} +---- +//NOTCONSOLE + +[source,js] +---- +POST /retrievers_example/_doc/1 +{ + "vector": [0.23, 0.67, 0.89], + "text": "Large language models are revolutionizing information retrieval by boosting search precision, deepening contextual understanding, and reshaping user experiences in data-rich environments.", + "year": 2024, + "topic": ["llm", "ai", "information_retrieval"] +} + +POST /retrievers_example/_doc/2 +{ + "vector": [0.12, 0.56, 0.78], + "text": "Artificial intelligence is transforming medicine, from advancing diagnostics and tailoring treatment plans to empowering predictive patient care for improved health outcomes.", + "year": 2023, + "topic": ["ai", "medicine"] +} + +POST /retrievers_example/_doc/3 +{ + "vector": [0.45, 0.32, 0.91], + "text": "AI is redefining security by enabling advanced threat detection, proactive risk analysis, and dynamic defenses against increasingly sophisticated cyber threats.", + "year": 2024, + "topic": ["ai", "security"] +} + +POST /retrievers_example/_doc/4 +{ + "vector": [0.34, 0.21, 0.98], + "text": "Elastic introduces Elastic AI Assistant, the open, generative AI sidekick powered by ESRE to democratize cybersecurity and enable users of every skill level.", + "year": 2023, + "topic": ["ai", "elastic", "assistant"] +} + +POST /retrievers_example/_doc/5 +{ + "vector": [0.11, 0.65, 0.47], + "text": "Learn how to spin up a deployment of our hosted Elasticsearch Service and use Elastic Observability to gain deeper insight into the behavior of your applications and systems.", + "year": 2024, + "topic": ["documentation", "observability", "elastic"] +} + +---- +//NOTCONSOLE + +Now that we also have our documents in place, let's try to run some queries using retrievers. + +[discrete] +[[retrievers-examples-combining-standard-knn-retrievers-with-rrf]] +==== Example: Combining query and kNN with RRF + +First, let's examine how to combine two different types of queries: a `kNN` query and a +`query_string` query. While these queries may produce scores in different ranges, we can use +Reciprocal Rank Fusion (`rrf`) to combine the results and generate a merged final result +list. + +To implement this in the retriever framework, we start with the top-level element: our `rrf` +retriever. This retriever operates on top of two other retrievers: a `knn` retriever and a +`standard` retriever. Our query structure would look like this: + +[source,js] +---- +GET /retrievers_example/_search +{ + "retriever":{ + "rrf": { + "retrievers":[ + { + "standard":{ + "query":{ + "query_string":{ + "query": "(information retrieval) OR (artificial intelligence)", + "default_field": "text" + } + } + } + }, + { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "_source": ["text", "topic"] +} +---- +//NOTCONSOLE + +[discrete] +[[retrievers-examples-collapsing-retriever-results]] +==== Example: Grouping results by year with `collapse` + +In our result set, we have many documents with the same `year` value. We can clean this +up using the `collapse` parameter with our retriever. This enables grouping results by +any field and returns only the highest-scoring document from each group. In this example +we'll collapse our results based on the `year` field. + +[source,js] +---- +GET /retrievers_example/_search +{ + "retriever":{ + "rrf": { + "retrievers":[ + { + "standard":{ + "query":{ + "query_string":{ + "query": "(information retrieval) OR (artificial intelligence)", + "default_field": "text" + } + } + } + }, + { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "collapse": { + "field": "year", + "inner_hits": { + "name": "topic related documents", + "_source": ["text", "year"] + } + }, + "_source": ["text", "topic"] +} +---- +//NOTCONSOLE + +[discrete] +[[retrievers-examples-text-similarity-reranker-on-top-of-rrf]] +==== Example: Rerank results of an RRF retriever + +Previously, we used a `text_similarity_reranker` retriever within an `rrf` retriever. +Because retrievers support full composability, we can also rerank the results of an +`rrf` retriever. Let's apply this to our first example. + +[source,js] +---- +GET retrievers_example/_search +{ + "retriever": { + "text_similarity_reranker": { + "retriever": { + "rrf": { + "retrievers": [ + { + "standard":{ + "query":{ + "query_string":{ + "query": "(information retrieval) OR (artificial intelligence)", + "default_field": "text" + } + } + } + }, + { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "field": "text", + "inference_id": "my-rerank-model", + "inference_text": "What are the state of the art applications of AI in information retrieval?" + } + }, + "_source": ["text", "topic"] +} + +---- +//NOTCONSOLE + +[discrete] +[[retrievers-examples-rrf-ranking-on-text-similarity-reranker-results]] +==== Example: RRF with semantic reranker + +For this example, we'll replace our semantic query with the `my-rerank-model` +reranker we previously configured. Since this is a reranker, it needs an initial pool of +documents to work with. In this case, we'll filter for documents about `ai` topics. + +[source,js] +---- +GET /retrievers_example/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + }, + { + "text_similarity_reranker": { + "retriever": { + "standard": { + "query": { + "term": { + "topic": "ai" + } + } + } + }, + "field": "text", + "inference_id": "my-rerank-model", + "inference_text": "Can I use generative AI to identify user intent and improve search relevance?" + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "_source": [ + "text", + "topic" + ] +} +---- +//NOTCONSOLE + +[discrete] +[[retrievers-examples-chaining-text-similarity-reranker-retrievers]] +==== Example: Chaining multiple semantic rerankers + +Full composability means we can chain together multiple retrievers of the same type. For instance, imagine we have a computationally expensive reranker that's specialized for AI content. We can rerank the results of a `text_similarity_reranker` using another `text_similarity_reranker` retriever. Each reranker can operate on different fields and/or use different inference services. + +[source,js] +---- +GET retrievers_example/_search +{ + "retriever": { + "text_similarity_reranker": { + "retriever": { + "text_similarity_reranker": { + "retriever": { + "knn": { + "field": "vector", + "query_vector": [ + 0.23, + 0.67, + 0.89 + ], + "k": 3, + "num_candidates": 5 + } + }, + "rank_window_size": 100, + "field": "text", + "inference_id": "my-rerank-model", + "inference_text": "What are the state of the art applications of AI in information retrieval?" + } + }, + "rank_window_size": 10, + "field": "text", + "inference_id": "my-other-more-expensive-rerank-model", + "inference_text": "Applications of Large Language Models in technology and their impact on user satisfaction" + } + }, + "_source": [ + "text", + "topic" + ] +} +---- +//NOTCONSOLE + + +Note that our example applies two reranking steps. First, we rerank the top 100 +documents from the `knn` search using the `my-rerank-model` reranker. Then we +pick the top 10 results and rerank them using the more fine-grained +`my-other-more-expensive-rerank-model`. + +[discrete] +[[retrievers-examples-rrf-and-aggregations]] +==== Example: Combine RRF with aggregations + +Retrievers support both composability and most of the standard `_search` functionality. For instance, +we can compute aggregations with the `rrf` retriever. When using a compound retriever, +the aggregations are computed based on its nested retrievers. In the following example, +the `terms` aggregation for the `topic` field will include all results, not just the top `rank_window_size`, +from the 2 nested retrievers, i.e. all documents whose `year` field is greater than 2023, and whose `topic` field +matches the term `elastic`. + +[source,js] +---- +GET retrievers_example/_search +{ + "retriever": { + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "range": { + "year": { + "gt": 2023 + } + } + } + } + }, + { + "standard": { + "query": { + "term": { + "topic": "elastic" + } + } + } + } + ], + "rank_window_size": 10, + "rank_constant": 1 + } + }, + "_source": [ + "text", + "topic" + ], + "aggs": { + "topics": { + "terms": { + "field": "topic" + } + } + } +} +---- +//NOTCONSOLE diff --git a/docs/reference/search/search-your-data/retrievers-overview.asciidoc b/docs/reference/search/search-your-data/retrievers-overview.asciidoc index 8e5955fc41782..1771b5bb0d849 100644 --- a/docs/reference/search/search-your-data/retrievers-overview.asciidoc +++ b/docs/reference/search/search-your-data/retrievers-overview.asciidoc @@ -1,5 +1,5 @@ [[retrievers-overview]] -=== Retrievers +== Retrievers A retriever is an abstraction that was added to the Search API in *8.14.0* and was made generally available in *8.16.0*. This abstraction enables the configuration of multi-stage retrieval pipelines within a single `_search` call. @@ -11,7 +11,7 @@ For implementation details, including notable restrictions, check out the [discrete] [[retrievers-overview-types]] -==== Retriever types +=== Retriever types Retrievers come in various types, each tailored for different search operations. The following retrievers are currently available: @@ -34,7 +34,8 @@ Used for <>. Requires first creating a `rerank` task using the <>. [discrete] -==== What makes retrievers useful? +[[retrievers-overview-why-are-they-useful]] +=== What makes retrievers useful? Here's an overview of what makes retrievers useful and how they differ from regular queries. @@ -66,65 +67,90 @@ When using compound retrievers, only the query element is allowed, which enforce [discrete] [[retrievers-overview-example]] -==== Example +=== Example -The following example demonstrates the powerful queries that we can now compose, and how retrievers simplify this process. -We can use any combination of retrievers we want, propagating the results of a nested retriever to its parent. -In this scenario, we'll make use of 4 of our currently available retrievers, i.e. `standard`, `knn`, `text_similarity_reranker` and `rrf`. -See <> for the complete list of available retrievers. - -We'll first combine the results of a `semantic` query using the `standard` retriever, and that of a `knn` search on a dense vector field, using `rrf` to get the top 100 results. -Finally, we'll then rerank the top-50 results of `rrf` using the `text_similarity_reranker` +The following example demonstrates how using retrievers simplify the composability of queries for RRF ranking. [source,js] ---- GET example-index/_search { "retriever": { - "text_similarity_reranker": { - "retriever": { - "rrf": { - "retrievers": [ - { - "standard": { - "query": { - "semantic": { - "field": "inference_field", - "query": "state of the art vector database" - } - } - } - }, - { - "knn": { - "query_vector": [ - 0.54, - ..., - 0.245 - ], - "field": "embedding", - "k": 10, - "num_candidates": 15 + "rrf": { + "retrievers": [ + { + "standard": { + "query": { + "sparse_vector": { + "field": "vector.tokens", + "inference_id": "my-elser-endpoint", + "query": "What blue shoes are on sale?" + } + } + } + }, + { + "standard": { + "query": { + "match": { + "text": "blue shoes sale" } } - ], - "rank_window_size": 100, - "rank_constant": 10 + } } - }, - "rank_window_size": 50, - "field": "description", - "inference_text": "what's the best way to create complex pipelines and retrieve documents?", - "inference_id": "my-awesome-rerank-model" + ] } } } ---- //NOTCONSOLE +This example demonstrates how you can combine different retrieval strategies into a single `retriever` pipeline. + +Compare to `RRF` with `sub_searches` approach (which is deprecated as of 8.16.0): + +.*Expand* for example +[%collapsible] +============== + +[source,js] +---- +GET example-index/_search +{ + "sub_searches":[ + { + "query":{ + "match":{ + "text":"blue shoes sale" + } + } + }, + { + "query":{ + "sparse_vector": { + "field": "vector.tokens", + "inference_id": "my-elser-endoint", + "query": "What blue shoes are on sale?" + } + } + } + ], + "rank":{ + "rrf":{ + "rank_window_size":50, + "rank_constant":20 + } + } +} +---- +//NOTCONSOLE +============== + +For more examples on how to use retrievers, please refer to <>. + [discrete] [[retrievers-overview-glossary]] -==== Glossary +=== Glossary Here are some important terms: @@ -143,7 +169,7 @@ Special compound retrievers that reorder hits and may adjust the number of hits, [discrete] [[retrievers-overview-play-in-search]] -==== Retrievers in action +=== Retrievers in action The Search Playground builds Elasticsearch queries using the retriever abstraction. It automatically detects the fields and types in your index and builds a retriever tree based on your selections. @@ -154,6 +180,9 @@ Refer to the {kibana-ref}/playground.html[Playground documentation] for more inf [discrete] [[retrievers-overview-api-reference]] -==== API reference +=== API reference For implementation details, including notable restrictions, check out the <> in the Search API docs. + + +include::retrievers-examples.asciidoc[] diff --git a/docs/reference/search/search-your-data/search-api.asciidoc b/docs/reference/search/search-your-data/search-api.asciidoc index 13cea537ea4fb..a9e74d54dd9d9 100644 --- a/docs/reference/search/search-your-data/search-api.asciidoc +++ b/docs/reference/search/search-your-data/search-api.asciidoc @@ -530,5 +530,4 @@ include::retrieve-inner-hits.asciidoc[] include::search-shard-routing.asciidoc[] include::search-using-query-rules.asciidoc[] include::search-template.asciidoc[] -include::retrievers-overview.asciidoc[] diff --git a/docs/reference/search/search-your-data/search-your-data.asciidoc b/docs/reference/search/search-your-data/search-your-data.asciidoc index cd2b418a7e79b..82541412db4bd 100644 --- a/docs/reference/search/search-your-data/search-your-data.asciidoc +++ b/docs/reference/search/search-your-data/search-your-data.asciidoc @@ -43,6 +43,7 @@ DSL, with a simplified user experience. Create search applications based on your results directly in the Kibana Search UI. include::search-api.asciidoc[] +include::retrievers-overview.asciidoc[] include::knn-search.asciidoc[] include::semantic-search.asciidoc[] include::search-across-clusters.asciidoc[] diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc index 370fc5c4ccf7e..58feb55f32e2f 100644 --- a/docs/reference/setup/install/docker.asciidoc +++ b/docs/reference/setup/install/docker.asciidoc @@ -86,6 +86,15 @@ docker run --name es01 --net elastic -p 9200:9200 -it -m 1GB {docker-image} TIP: Use the `-m` flag to set a memory limit for the container. This removes the need to <>. + + +{ml-cap} features such as <> +require a larger container with more than 1GB of memory. +If you intend to use the {ml} capabilities, then start the container with this command: ++ +[source,sh,subs="attributes"] +---- +docker run --name es01 --net elastic -p 9200:9200 -it -m 6GB -e "xpack.ml.use_auto_machine_memory_percent=true" {docker-image} +---- The command prints the `elastic` user password and an enrollment token for {kib}. . Copy the generated `elastic` password and enrollment token. These credentials diff --git a/docs/reference/setup/install/docker/docker-compose.yml b/docs/reference/setup/install/docker/docker-compose.yml index 15d8c11e2f12f..db5b6e6c91b49 100644 --- a/docs/reference/setup/install/docker/docker-compose.yml +++ b/docs/reference/setup/install/docker/docker-compose.yml @@ -90,6 +90,7 @@ services: - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt - xpack.security.transport.ssl.verification_mode=certificate - xpack.license.self_generated.type=${LICENSE} + - xpack.ml.use_auto_machine_memory_percent=true mem_limit: ${MEM_LIMIT} ulimits: memlock: @@ -130,6 +131,7 @@ services: - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt - xpack.security.transport.ssl.verification_mode=certificate - xpack.license.self_generated.type=${LICENSE} + - xpack.ml.use_auto_machine_memory_percent=true mem_limit: ${MEM_LIMIT} ulimits: memlock: @@ -170,6 +172,7 @@ services: - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt - xpack.security.transport.ssl.verification_mode=certificate - xpack.license.self_generated.type=${LICENSE} + - xpack.ml.use_auto_machine_memory_percent=true mem_limit: ${MEM_LIMIT} ulimits: memlock: diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle index 6f127b1074c68..eff7c950e32a2 100644 --- a/modules/analysis-common/build.gradle +++ b/modules/analysis-common/build.gradle @@ -6,7 +6,7 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' @@ -30,6 +30,15 @@ dependencies { clusterModules project(':modules:mapper-extras') } +tasks.named("yamlRestTest").configure { task-> + if (BuildParams.getRuntimeJavaVersion().majorVersion.toInteger() < 21) { + // Requires at least Java 21 + systemProperty 'tests.rest.blacklist', [ + "analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)" + ].join(',') + } +} + tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTest("indices.analyze/10_analyze/htmlStrip_deprecated", "Cleanup versioned deprecations in analysis #41560") task.skipTest("analysis-common/40_token_filters/delimited_payload_filter_error", "Remove preconfigured delimited_payload_filter #43686") diff --git a/muted-tests.yml b/muted-tests.yml index 273191628bae6..936150a403659 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -246,17 +246,11 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/115664 - class: org.elasticsearch.indices.mapping.UpdateMappingIntegrationIT issue: https://github.com/elastic/elasticsearch/issues/116126 -- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT - method: test {yaml=logsdb/10_settings/logsdb with default ignore dynamic beyond limit and non-default sorting} - issue: https://github.com/elastic/elasticsearch/issues/116095 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT issue: https://github.com/elastic/elasticsearch/issues/111319 - class: org.elasticsearch.upgrades.FullClusterRestartIT method: testSnapshotRestore {cluster=OLD} issue: https://github.com/elastic/elasticsearch/issues/111777 -- class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT - method: test {yaml=analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)} - issue: https://github.com/elastic/elasticsearch/issues/116134 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT method: testLookbackWithIndicesOptions issue: https://github.com/elastic/elasticsearch/issues/116127 @@ -287,9 +281,6 @@ tests: - class: org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderIT method: testEnterpriseDownloaderTask issue: https://github.com/elastic/elasticsearch/issues/115163 -- class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityReloadCredentialsRestIT - method: testFirstTimeSetupWithElasticsearchSettings - issue: https://github.com/elastic/elasticsearch/issues/116286 - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT method: testEveryActionIsEitherOperatorOnlyOrNonOperator issue: https://github.com/elastic/elasticsearch/issues/102992 @@ -299,9 +290,6 @@ tests: - class: org.elasticsearch.search.basic.SearchWhileRelocatingIT method: testSearchAndRelocateConcurrentlyRandomReplicas issue: https://github.com/elastic/elasticsearch/issues/116145 -- class: org.elasticsearch.xpack.inference.InferenceRestIT - method: test {p0=inference/40_semantic_text_query/Query a field that uses the default ELSER 2 endpoint} - issue: https://github.com/elastic/elasticsearch/issues/114376 - class: org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests method: testHasPrivilegesOtherThanIndex issue: https://github.com/elastic/elasticsearch/issues/116376 @@ -313,6 +301,38 @@ tests: - class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT method: test {yaml=/10_apm/Test template reinstallation} issue: https://github.com/elastic/elasticsearch/issues/116445 +- class: org.elasticsearch.action.admin.HotThreadsIT + method: testHotThreadsDontFail + issue: https://github.com/elastic/elasticsearch/issues/115754 +- class: org.elasticsearch.action.search.PointInTimeIT + method: testPITTiebreak + issue: https://github.com/elastic/elasticsearch/issues/115810 +- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT + method: test {p0=esql/61_enrich_ip/IP strings} + issue: https://github.com/elastic/elasticsearch/issues/116529 +- class: org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsCanMatchOnCoordinatorIntegTests + method: testSearchableSnapshotShardsAreSkippedBySearchRequestWithoutQueryingAnyNodeWhenTheyAreOutsideOfTheQueryRange + issue: https://github.com/elastic/elasticsearch/issues/116523 +- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT + method: testInferDeploysDefaultElser + issue: https://github.com/elastic/elasticsearch/issues/114913 +- class: org.elasticsearch.threadpool.SimpleThreadPoolIT + method: testThreadPoolMetrics + issue: https://github.com/elastic/elasticsearch/issues/108320 +- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT + method: test {p0=esql/60_enrich/Enrich on keyword with fields alias} + issue: https://github.com/elastic/elasticsearch/issues/116592 +- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT + method: test {p0=esql/60_enrich/Enrich on keyword with fields} + issue: https://github.com/elastic/elasticsearch/issues/116593 +- class: org.elasticsearch.xpack.kql.query.KqlQueryBuilderTests + issue: https://github.com/elastic/elasticsearch/issues/116487 +- class: org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissionsTests + method: testCollapseAndRemoveUnsupportedPrivileges + issue: https://github.com/elastic/elasticsearch/issues/116520 +- class: org.elasticsearch.xpack.spatial.search.GeoGridAggAndQueryConsistencyIT + method: testGeoShapeGeoTile + issue: https://github.com/elastic/elasticsearch/issues/115717 # Examples: # diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml new file mode 100644 index 0000000000000..161fc23a84651 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml @@ -0,0 +1,163 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ dense_vector_docvalue_fields ] + test_runner_features: [ capabilities, close_to ] + reason: Capability required to run test + - do: + indices.create: + index: test + body: + mappings: + properties: + name: + type: keyword + vector1: + type: dense_vector + element_type: float + dims: 5 + index: true + vector2: + type: dense_vector + element_type: float + dims: 5 + index: false + vector3: + type: dense_vector + element_type: byte + dims: 5 + index: true + vector4: + type: dense_vector + element_type: byte + dims: 5 + index: false + vector5: + type: dense_vector + element_type: bit + dims: 40 + index: true + vector6: + type: dense_vector + element_type: bit + dims: 40 + index: false + - do: + index: + index: test + id: "1" + body: + name: cow.jpg + vector1: [230.0, 300.33, -34.8988, 15.555, -200.0] + vector2: [130.0, 115.0, -1.02, 15.555, -100.0] + vector3: [-1, 100, -13, 15, -128] + vector4: [-1, 50, -1, 1, 120] + vector5: [1, 111, -13, 15, -128] + vector6: [-1, 11, 0, 12, 111] + - do: + index: + index: test + id: "2" + body: + name: moose.jpg + vector1: [-0.5, 100.0, -13, 14.8, -156.0] + vector4: [-1, 50, -1, 1, 120] + vector5: [1, 111, -13, 15, -128] + vector6: null + - do: + index: + index: test + id: "3" + body: + name: rabbit.jpg + vector2: [130.0, 115.0, -1.02, 15.555, -100.0] + vector3: [-1, 100, -13, 15, -128] + + - do: + indices.refresh: {} + +--- +"Enable docvalue_fields parameter for dense_vector fields": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ dense_vector_docvalue_fields ] + test_runner_features: capabilities + reason: "Support for dense vector doc value fields capability required" + - do: + search: + _source: false + index: test + body: + docvalue_fields: [name, vector1, vector2, vector3, vector4, vector5, vector6] + sort: name + + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0.fields.name.0: "cow.jpg"} + + - length: {hits.hits.0.fields.vector1.0: 5} + - length: {hits.hits.0.fields.vector2.0: 5} + - length: {hits.hits.0.fields.vector3.0: 5} + - length: {hits.hits.0.fields.vector4.0: 5} + - length: {hits.hits.0.fields.vector5.0: 5} + - length: {hits.hits.0.fields.vector6.0: 5} + + - close_to: { hits.hits.0.fields.vector1.0.0: { value: 230.0, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector1.0.1: { value: 300.33, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector1.0.2: { value: -34.8988, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector1.0.3: { value: 15.555, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector1.0.4: { value: -200.0, error: 0.001 } } + + - close_to: { hits.hits.0.fields.vector2.0.0: { value: 130.0, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector2.0.1: { value: 115.0, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector2.0.2: { value: -1.02, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector2.0.3: { value: 15.555, error: 0.001 } } + - close_to: { hits.hits.0.fields.vector2.0.4: { value: -100.0, error: 0.001 } } + + - match: {hits.hits.0.fields.vector3.0: [-1, 100, -13, 15, -128]} + - match: {hits.hits.0.fields.vector4.0: [-1, 50, -1, 1, 120]} + - match: {hits.hits.0.fields.vector5.0: [1, 111, -13, 15, -128]} + - match: {hits.hits.0.fields.vector6.0: [-1, 11, 0, 12, 111]} + + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} + + - length: {hits.hits.1.fields.vector1.0: 5} + - length: {hits.hits.1.fields.vector4.0: 5} + - length: {hits.hits.1.fields.vector5.0: 5} + - match: {hits.hits.1.fields.vector2: null} + - match: {hits.hits.1.fields.vector3: null} + - match: {hits.hits.1.fields.vector6: null} + + - close_to: { hits.hits.1.fields.vector1.0.0: { value: -0.5, error: 0.001 } } + - close_to: { hits.hits.1.fields.vector1.0.1: { value: 100.0, error: 0.001 } } + - close_to: { hits.hits.1.fields.vector1.0.2: { value: -13, error: 0.001 } } + - close_to: { hits.hits.1.fields.vector1.0.3: { value: 14.8, error: 0.001 } } + - close_to: { hits.hits.1.fields.vector1.0.4: { value: -156.0, error: 0.001 } } + + - match: {hits.hits.1.fields.vector4.0: [-1, 50, -1, 1, 120]} + - match: {hits.hits.1.fields.vector5.0: [1, 111, -13, 15, -128]} + + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} + + - length: {hits.hits.2.fields.vector2.0: 5} + - length: {hits.hits.2.fields.vector3.0: 5} + - match: {hits.hits.2.fields.vector1: null} + - match: {hits.hits.2.fields.vector4: null} + - match: {hits.hits.2.fields.vector5: null} + - match: {hits.hits.2.fields.vector6: null} + + - close_to: { hits.hits.2.fields.vector2.0.0: { value: 130.0, error: 0.001 } } + - close_to: { hits.hits.2.fields.vector2.0.1: { value: 115.0, error: 0.001 } } + - close_to: { hits.hits.2.fields.vector2.0.2: { value: -1.02, error: 0.001 } } + - close_to: { hits.hits.2.fields.vector2.0.3: { value: 15.555, error: 0.001 } } + - close_to: { hits.hits.2.fields.vector2.0.4: { value: -100.0, error: 0.001 } } + + - match: {hits.hits.2.fields.vector3.0: [-1, 100, -13, 15, -128]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_multi_dense_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_multi_dense_vector.yml new file mode 100644 index 0000000000000..80d1d25dfcbd8 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/30_multi_dense_vector.yml @@ -0,0 +1,141 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ multi_dense_vector_field_mapper ] + test_runner_features: capabilities + reason: "Support for multi dense vector field mapper capability required" +--- +"Test create multi-vector field": + - do: + indices.create: + index: test + body: + mappings: + properties: + vector1: + type: multi_dense_vector + dims: 3 + - do: + index: + index: test + id: "1" + body: + vector1: [[2, -1, 1]] + - do: + index: + index: test + id: "2" + body: + vector1: [[2, -1, 1], [3, 4, 5]] + - do: + index: + index: test + id: "3" + body: + vector1: [[2, -1, 1], [3, 4, 5], [6, 7, 8]] + - do: + indices.refresh: {} +--- +"Test create dynamic dim multi-vector field": + - do: + indices.create: + index: test + body: + mappings: + properties: + name: + type: keyword + vector1: + type: multi_dense_vector + - do: + index: + index: test + id: "1" + body: + vector1: [[2, -1, 1]] + - do: + index: + index: test + id: "2" + body: + vector1: [[2, -1, 1], [3, 4, 5]] + - do: + index: + index: test + id: "3" + body: + vector1: [[2, -1, 1], [3, 4, 5], [6, 7, 8]] + - do: + cluster.health: + wait_for_events: languid + + # verify some other dimension will fail + - do: + catch: bad_request + index: + index: test + id: "4" + body: + vector1: [[2, -1, 1], [3, 4, 5], [6, 7, 8, 9]] +--- +"Test dynamic dim mismatch fails multi-vector field": + - do: + indices.create: + index: test + body: + mappings: + properties: + vector1: + type: multi_dense_vector + - do: + catch: bad_request + index: + index: test + id: "1" + body: + vector1: [[2, -1, 1], [2]] +--- +"Test static dim mismatch fails multi-vector field": + - do: + indices.create: + index: test + body: + mappings: + properties: + vector1: + type: multi_dense_vector + dims: 3 + - do: + catch: bad_request + index: + index: test + id: "1" + body: + vector1: [[2, -1, 1], [2]] +--- +"Test poorly formatted multi-vector field": + - do: + indices.create: + index: poorly_formatted_vector + body: + mappings: + properties: + vector1: + type: multi_dense_vector + dims: 3 + - do: + catch: bad_request + index: + index: poorly_formatted_vector + id: "1" + body: + vector1: [[[2, -1, 1]]] + - do: + catch: bad_request + index: + index: poorly_formatted_vector + id: "1" + body: + vector1: [[2, -1, 1], [[2, -1, 1]]] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 871e5aa48e3e4..dc63d6827af24 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -188,6 +188,8 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_CCS_EXEC_INFO_WITH_FAILURES = def(8_783_00_0); public static final TransportVersion LOGSDB_TELEMETRY = def(8_784_00_0); public static final TransportVersion LOGSDB_TELEMETRY_STATS = def(8_785_00_0); + public static final TransportVersion KQL_QUERY_ADDED = def(8_786_00_0); + public static final TransportVersion ROLE_MONITOR_STATS = def(8_787_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 97a259cc6f030..fb672b49c2f5a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -18,6 +18,7 @@ import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.cluster.ClusterState; @@ -44,6 +45,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -455,11 +457,12 @@ private TokenCounter(int maxTokenCount) { private void increment() { tokenCount++; if (tokenCount > maxTokenCount) { - throw new IllegalStateException( + throw new ElasticsearchStatusException( "The number of tokens produced by calling _analyze has exceeded the allowed maximum of [" + maxTokenCount + "]." - + " This limit can be set by changing the [index.analyze.max_token_count] index level setting." + + " This limit can be set by changing the [index.analyze.max_token_count] index level setting.", + RestStatus.BAD_REQUEST ); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java index 8ce2cc7b6b19e..a93f763933eeb 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java @@ -132,7 +132,6 @@ private static boolean assertSearchCoordinationThread() { @Override public void run() { assert assertSearchCoordinationThread(); - checkNoMissingShards(); Version version = request.minCompatibleShardNode(); if (version != null && Version.CURRENT.minimumCompatibilityVersion().equals(version) == false) { if (checkMinimumVersion(shardsIts) == false) { @@ -185,7 +184,10 @@ private void runCoordinatorRewritePhase() { if (matchedShardLevelRequests.isEmpty()) { finishPhase(); } else { - new Round(new GroupShardsIterator<>(matchedShardLevelRequests)).run(); + GroupShardsIterator matchingShards = new GroupShardsIterator<>(matchedShardLevelRequests); + // verify missing shards only for the shards that we hit for the query + checkNoMissingShards(matchingShards); + new Round(matchingShards).run(); } } @@ -195,9 +197,9 @@ private void consumeResult(boolean canMatch, ShardSearchRequest request) { results.consumeResult(result, () -> {}); } - private void checkNoMissingShards() { + private void checkNoMissingShards(GroupShardsIterator shards) { assert assertSearchCoordinationThread(); - doCheckNoMissingShards(getName(), request, shardsIts); + doCheckNoMissingShards(getName(), request, shards); } private Map> groupByNode(GroupShardsIterator shards) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 39499253c8790..bf80c38d64a4e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -402,8 +402,10 @@ Index[] concreteIndices(Context context, String... indexExpressions) { resolveIndicesForDataStream(context, dataStream, concreteIndicesResult); } } else { - for (Index index : indexAbstraction.getIndices()) { - if (shouldTrackConcreteIndex(context, context.getOptions(), index)) { + List indices = indexAbstraction.getIndices(); + for (int i = 0, n = indices.size(); i < n; i++) { + Index index = indices.get(i); + if (shouldTrackConcreteIndex(context, index)) { concreteIndicesResult.add(index); } } @@ -421,7 +423,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) { private static void resolveIndicesForDataStream(Context context, DataStream dataStream, Set concreteIndicesResult) { if (shouldIncludeRegularIndices(context.getOptions())) { for (Index index : dataStream.getIndices()) { - if (shouldTrackConcreteIndex(context, context.getOptions(), index)) { + if (shouldTrackConcreteIndex(context, index)) { concreteIndicesResult.add(index); } } @@ -430,7 +432,7 @@ private static void resolveIndicesForDataStream(Context context, DataStream data // We short-circuit here, if failure indices are not allowed and they can be skipped if (context.getOptions().allowFailureIndices() || context.getOptions().ignoreUnavailable() == false) { for (Index index : dataStream.getFailureIndices().getIndices()) { - if (shouldTrackConcreteIndex(context, context.getOptions(), index)) { + if (shouldTrackConcreteIndex(context, index)) { concreteIndicesResult.add(index); } } @@ -565,7 +567,7 @@ private static IndexNotFoundException notFoundException(String... indexExpressio return infe; } - private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions options, Index index) { + private static boolean shouldTrackConcreteIndex(Context context, Index index) { if (context.systemIndexAccessLevel == SystemIndexAccessLevel.BACKWARDS_COMPATIBLE_ONLY && context.netNewSystemIndexPredicate.test(index.getName())) { // Exclude this one as it's a net-new system index, and we explicitly don't want those. @@ -575,7 +577,7 @@ private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions DataStream parentDataStream = context.getState().metadata().getIndicesLookup().get(index.getName()).getParentDataStream(); if (parentDataStream != null && parentDataStream.isFailureStoreEnabled()) { if (parentDataStream.isFailureStoreIndex(index.getName())) { - if (options.ignoreUnavailable()) { + if (context.options.ignoreUnavailable()) { return false; } else { throw new FailureIndexNotSupportedException(index); @@ -585,6 +587,7 @@ private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions } final IndexMetadata imd = context.state.metadata().index(index); if (imd.getState() == IndexMetadata.State.CLOSE) { + IndicesOptions options = context.options; if (options.forbidClosedIndices() && options.ignoreUnavailable() == false) { throw new IndexClosedException(index); } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index a023837a0efb7..582f5f1d3e881 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -415,13 +415,18 @@ public double computeSquaredMagnitude(VectorData vectorData) { return VectorUtil.dotProduct(vectorData.asByteVector(), vectorData.asByteVector()); } - private VectorData parseVectorArray(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException { + private VectorData parseVectorArray( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { int index = 0; - byte[] vector = new byte[fieldMapper.fieldType().dims]; + byte[] vector = new byte[dims]; float squaredMagnitude = 0; for (XContentParser.Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser() .nextToken()) { - fieldMapper.checkDimensionExceeded(index, context); + dimChecker.accept(index, false); ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser()); final int value; if (context.parser().numberType() != XContentParser.NumberType.INT) { @@ -459,30 +464,31 @@ private VectorData parseVectorArray(DocumentParserContext context, DenseVectorFi vector[index++] = (byte) value; squaredMagnitude += value * value; } - fieldMapper.checkDimensionMatches(index, context); - checkVectorMagnitude(fieldMapper.fieldType().similarity, errorByteElementsAppender(vector), squaredMagnitude); + dimChecker.accept(index, true); + checkVectorMagnitude(similarity, errorByteElementsAppender(vector), squaredMagnitude); return VectorData.fromBytes(vector); } - private VectorData parseHexEncodedVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException { + private VectorData parseHexEncodedVector( + DocumentParserContext context, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { byte[] decodedVector = HexFormat.of().parseHex(context.parser().text()); - fieldMapper.checkDimensionMatches(decodedVector.length, context); + dimChecker.accept(decodedVector.length, true); VectorData vectorData = VectorData.fromBytes(decodedVector); double squaredMagnitude = computeSquaredMagnitude(vectorData); - checkVectorMagnitude( - fieldMapper.fieldType().similarity, - errorByteElementsAppender(decodedVector), - (float) squaredMagnitude - ); + checkVectorMagnitude(similarity, errorByteElementsAppender(decodedVector), (float) squaredMagnitude); return vectorData; } @Override - VectorData parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException { + VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) + throws IOException { XContentParser.Token token = context.parser().currentToken(); return switch (token) { - case START_ARRAY -> parseVectorArray(context, fieldMapper); - case VALUE_STRING -> parseHexEncodedVector(context, fieldMapper); + case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity); + case VALUE_STRING -> parseHexEncodedVector(context, dimChecker, similarity); default -> throw new ParsingException( context.parser().getTokenLocation(), format("Unsupported type [%s] for provided value [%s]", token, context.parser().text()) @@ -492,7 +498,13 @@ VectorData parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper @Override public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException { - VectorData vectorData = parseKnnVector(context, fieldMapper); + VectorData vectorData = parseKnnVector(context, fieldMapper.fieldType().dims, (i, end) -> { + if (end) { + fieldMapper.checkDimensionMatches(i, context); + } else { + fieldMapper.checkDimensionExceeded(i, context); + } + }, fieldMapper.fieldType().similarity); Field field = createKnnVectorField( fieldMapper.fieldType().name(), vectorData.asByteVector(), @@ -676,21 +688,22 @@ && isNotUnitVector(squaredMagnitude)) { } @Override - VectorData parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException { + VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) + throws IOException { int index = 0; float squaredMagnitude = 0; - float[] vector = new float[fieldMapper.fieldType().dims]; + float[] vector = new float[dims]; for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) { - fieldMapper.checkDimensionExceeded(index, context); + dimChecker.accept(index, false); ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser()); float value = context.parser().floatValue(true); vector[index] = value; squaredMagnitude += value * value; index++; } - fieldMapper.checkDimensionMatches(index, context); + dimChecker.accept(index, true); checkVectorBounds(vector); - checkVectorMagnitude(fieldMapper.fieldType().similarity, errorFloatElementsAppender(vector), squaredMagnitude); + checkVectorMagnitude(similarity, errorFloatElementsAppender(vector), squaredMagnitude); return VectorData.fromFloats(vector); } @@ -815,12 +828,17 @@ public double computeSquaredMagnitude(VectorData vectorData) { return count; } - private VectorData parseVectorArray(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException { + private VectorData parseVectorArray( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException { int index = 0; - byte[] vector = new byte[fieldMapper.fieldType().dims / Byte.SIZE]; + byte[] vector = new byte[dims / Byte.SIZE]; for (XContentParser.Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser() .nextToken()) { - fieldMapper.checkDimensionExceeded(index, context); + dimChecker.accept(index * Byte.SIZE, false); ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser()); final int value; if (context.parser().numberType() != XContentParser.NumberType.INT) { @@ -855,35 +873,25 @@ private VectorData parseVectorArray(DocumentParserContext context, DenseVectorFi + "];" ); } - if (index >= vector.length) { - throw new IllegalArgumentException( - "The number of dimensions for field [" - + fieldMapper.fieldType().name() - + "] should be [" - + fieldMapper.fieldType().dims - + "] but found [" - + (index + 1) * Byte.SIZE - + "]" - ); - } vector[index++] = (byte) value; } - fieldMapper.checkDimensionMatches(index * Byte.SIZE, context); + dimChecker.accept(index * Byte.SIZE, true); return VectorData.fromBytes(vector); } - private VectorData parseHexEncodedVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException { + private VectorData parseHexEncodedVector(DocumentParserContext context, IntBooleanConsumer dimChecker) throws IOException { byte[] decodedVector = HexFormat.of().parseHex(context.parser().text()); - fieldMapper.checkDimensionMatches(decodedVector.length * Byte.SIZE, context); + dimChecker.accept(decodedVector.length * Byte.SIZE, true); return VectorData.fromBytes(decodedVector); } @Override - VectorData parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException { + VectorData parseKnnVector(DocumentParserContext context, int dims, IntBooleanConsumer dimChecker, VectorSimilarity similarity) + throws IOException { XContentParser.Token token = context.parser().currentToken(); return switch (token) { - case START_ARRAY -> parseVectorArray(context, fieldMapper); - case VALUE_STRING -> parseHexEncodedVector(context, fieldMapper); + case START_ARRAY -> parseVectorArray(context, dims, dimChecker, similarity); + case VALUE_STRING -> parseHexEncodedVector(context, dimChecker); default -> throw new ParsingException( context.parser().getTokenLocation(), format("Unsupported type [%s] for provided value [%s]", token, context.parser().text()) @@ -893,7 +901,13 @@ VectorData parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper @Override public void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException { - VectorData vectorData = parseKnnVector(context, fieldMapper); + VectorData vectorData = parseKnnVector(context, fieldMapper.fieldType().dims, (i, end) -> { + if (end) { + fieldMapper.checkDimensionMatches(i, context); + } else { + fieldMapper.checkDimensionExceeded(i, context); + } + }, fieldMapper.fieldType().similarity); Field field = createKnnVectorField( fieldMapper.fieldType().name(), vectorData.asByteVector(), @@ -957,7 +971,12 @@ public void checkDimensions(Integer dvDims, int qvDims) { abstract void parseKnnVectorAndIndex(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException; - abstract VectorData parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException; + abstract VectorData parseKnnVector( + DocumentParserContext context, + int dims, + IntBooleanConsumer dimChecker, + VectorSimilarity similarity + ) throws IOException; abstract int getNumBytes(int dimensions); @@ -1903,9 +1922,7 @@ protected Object parseSourceValue(Object value) { @Override public DocValueFormat docValueFormat(String format, ZoneId timeZone) { - throw new IllegalArgumentException( - "Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations" - ); + return DocValueFormat.DENSE_VECTOR; } @Override @@ -2181,7 +2198,13 @@ private void parseBinaryDocValuesVectorAndIndex(DocumentParserContext context) t : elementType.getNumBytes(dims); ByteBuffer byteBuffer = elementType.createByteBuffer(indexCreatedVersion, numBytes); - VectorData vectorData = elementType.parseKnnVector(context, this); + VectorData vectorData = elementType.parseKnnVector(context, dims, (i, b) -> { + if (b) { + checkDimensionMatches(i, context); + } else { + checkDimensionExceeded(i, context); + } + }, fieldType().similarity); vectorData.addToBuffer(byteBuffer); if (indexCreatedVersion.onOrAfter(MAGNITUDE_STORED_INDEX_VERSION)) { // encode vector magnitude at the end @@ -2429,4 +2452,11 @@ public String fieldName() { return fullPath(); } } + + /** + * @FunctionalInterface for a function that takes a int and boolean + */ + interface IntBooleanConsumer { + void accept(int value, boolean isComplete); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapper.java new file mode 100644 index 0000000000000..b23a1f1f66792 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapper.java @@ -0,0 +1,431 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.vectors; + +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.FieldExistsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.FeatureFlag; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.fielddata.FieldDataContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.ArraySourceValueFetcher; +import org.elasticsearch.index.mapper.DocumentParserContext; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.SimpleMappedFieldType; +import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.vectors.VectorData; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT_BIT; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.namesToElementType; + +public class MultiDenseVectorFieldMapper extends FieldMapper { + + public static final String VECTOR_MAGNITUDES_SUFFIX = "._magnitude"; + public static final FeatureFlag FEATURE_FLAG = new FeatureFlag("multi_dense_vector"); + public static final String CONTENT_TYPE = "multi_dense_vector"; + + private static MultiDenseVectorFieldMapper toType(FieldMapper in) { + return (MultiDenseVectorFieldMapper) in; + } + + public static class Builder extends FieldMapper.Builder { + + private final Parameter elementType = new Parameter<>( + "element_type", + false, + () -> DenseVectorFieldMapper.ElementType.FLOAT, + (n, c, o) -> { + DenseVectorFieldMapper.ElementType elementType = namesToElementType.get((String) o); + if (elementType == null) { + throw new MapperParsingException( + "invalid element_type [" + o + "]; available types are " + namesToElementType.keySet() + ); + } + return elementType; + }, + m -> toType(m).fieldType().elementType, + XContentBuilder::field, + Objects::toString + ); + + // This is defined as updatable because it can be updated once, from [null] to a valid dim size, + // by a dynamic mapping update. Once it has been set, however, the value cannot be changed. + private final Parameter dims = new Parameter<>("dims", true, () -> null, (n, c, o) -> { + if (o instanceof Integer == false) { + throw new MapperParsingException("Property [dims] on field [" + n + "] must be an integer but got [" + o + "]"); + } + + return XContentMapValues.nodeIntegerValue(o); + }, m -> toType(m).fieldType().dims, XContentBuilder::field, Object::toString).setSerializerCheck((id, ic, v) -> v != null) + .setMergeValidator((previous, current, c) -> previous == null || Objects.equals(previous, current)) + .addValidator(dims -> { + if (dims == null) { + return; + } + int maxDims = elementType.getValue() == DenseVectorFieldMapper.ElementType.BIT ? MAX_DIMS_COUNT_BIT : MAX_DIMS_COUNT; + int minDims = elementType.getValue() == DenseVectorFieldMapper.ElementType.BIT ? Byte.SIZE : 1; + if (dims < minDims || dims > maxDims) { + throw new MapperParsingException( + "The number of dimensions should be in the range [" + minDims + ", " + maxDims + "] but was [" + dims + "]" + ); + } + if (elementType.getValue() == DenseVectorFieldMapper.ElementType.BIT) { + if (dims % Byte.SIZE != 0) { + throw new MapperParsingException("The number of dimensions for should be a multiple of 8 but was [" + dims + "]"); + } + } + }); + private final Parameter> meta = Parameter.metaParam(); + + private final IndexVersion indexCreatedVersion; + + public Builder(String name, IndexVersion indexCreatedVersion) { + super(name); + this.indexCreatedVersion = indexCreatedVersion; + } + + @Override + protected Parameter[] getParameters() { + return new Parameter[] { elementType, dims, meta }; + } + + public MultiDenseVectorFieldMapper.Builder dimensions(int dimensions) { + this.dims.setValue(dimensions); + return this; + } + + public MultiDenseVectorFieldMapper.Builder elementType(DenseVectorFieldMapper.ElementType elementType) { + this.elementType.setValue(elementType); + return this; + } + + @Override + public MultiDenseVectorFieldMapper build(MapperBuilderContext context) { + // Validate again here because the dimensions or element type could have been set programmatically, + // which affects index option validity + validate(); + return new MultiDenseVectorFieldMapper( + leafName(), + new MultiDenseVectorFieldType( + context.buildFullName(leafName()), + elementType.getValue(), + dims.getValue(), + indexCreatedVersion, + meta.getValue() + ), + builderParams(this, context), + indexCreatedVersion + ); + } + } + + public static final TypeParser PARSER = new TypeParser( + (n, c) -> new MultiDenseVectorFieldMapper.Builder(n, c.indexVersionCreated()), + notInMultiFields(CONTENT_TYPE) + ); + + public static final class MultiDenseVectorFieldType extends SimpleMappedFieldType { + private final DenseVectorFieldMapper.ElementType elementType; + private final Integer dims; + private final IndexVersion indexCreatedVersion; + + public MultiDenseVectorFieldType( + String name, + DenseVectorFieldMapper.ElementType elementType, + Integer dims, + IndexVersion indexCreatedVersion, + Map meta + ) { + super(name, false, false, true, TextSearchInfo.NONE, meta); + this.elementType = elementType; + this.dims = dims; + this.indexCreatedVersion = indexCreatedVersion; + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + return new ArraySourceValueFetcher(name(), context) { + @Override + protected Object parseSourceValue(Object value) { + return value; + } + }; + } + + @Override + public DocValueFormat docValueFormat(String format, ZoneId timeZone) { + throw new IllegalArgumentException( + "Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations" + ); + } + + @Override + public boolean isAggregatable() { + return false; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { + return new MultiVectorIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, indexCreatedVersion, dims, elementType); + } + + @Override + public Query existsQuery(SearchExecutionContext context) { + return new FieldExistsQuery(name()); + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support term queries"); + } + + int getVectorDimensions() { + return dims; + } + + DenseVectorFieldMapper.ElementType getElementType() { + return elementType; + } + } + + private final IndexVersion indexCreatedVersion; + + private MultiDenseVectorFieldMapper( + String simpleName, + MappedFieldType fieldType, + BuilderParams params, + IndexVersion indexCreatedVersion + ) { + super(simpleName, fieldType, params); + this.indexCreatedVersion = indexCreatedVersion; + } + + @Override + public MultiDenseVectorFieldType fieldType() { + return (MultiDenseVectorFieldType) super.fieldType(); + } + + @Override + public boolean parsesArrayValue() { + return true; + } + + @Override + public void parse(DocumentParserContext context) throws IOException { + if (context.doc().getByKey(fieldType().name()) != null) { + throw new IllegalArgumentException( + "Field [" + + fullPath() + + "] of type [" + + typeName() + + "] doesn't support indexing multiple values for the same field in the same document" + ); + } + if (XContentParser.Token.VALUE_NULL == context.parser().currentToken()) { + return; + } + if (XContentParser.Token.START_ARRAY != context.parser().currentToken()) { + throw new IllegalArgumentException( + "Field [" + fullPath() + "] of type [" + typeName() + "] cannot be indexed with a single value" + ); + } + if (fieldType().dims == null) { + int currentDims = -1; + while (XContentParser.Token.END_ARRAY != context.parser().nextToken()) { + int dims = fieldType().elementType.parseDimensionCount(context); + if (currentDims == -1) { + currentDims = dims; + } else if (currentDims != dims) { + throw new IllegalArgumentException( + "Field [" + fullPath() + "] of type [" + typeName() + "] cannot be indexed with vectors of different dimensions" + ); + } + } + MultiDenseVectorFieldType updatedFieldType = new MultiDenseVectorFieldType( + fieldType().name(), + fieldType().elementType, + currentDims, + indexCreatedVersion, + fieldType().meta() + ); + Mapper update = new MultiDenseVectorFieldMapper(leafName(), updatedFieldType, builderParams, indexCreatedVersion); + context.addDynamicMapper(update); + return; + } + int dims = fieldType().dims; + DenseVectorFieldMapper.ElementType elementType = fieldType().elementType; + List vectors = new ArrayList<>(); + while (XContentParser.Token.END_ARRAY != context.parser().nextToken()) { + VectorData vector = elementType.parseKnnVector(context, dims, (i, b) -> { + if (b) { + checkDimensionMatches(i, context); + } else { + checkDimensionExceeded(i, context); + } + }, null); + vectors.add(vector); + } + int bufferSize = elementType.getNumBytes(dims) * vectors.size(); + ByteBuffer buffer = ByteBuffer.allocate(bufferSize).order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer magnitudeBuffer = ByteBuffer.allocate(vectors.size() * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN); + for (VectorData vector : vectors) { + vector.addToBuffer(buffer); + magnitudeBuffer.putFloat((float) Math.sqrt(elementType.computeSquaredMagnitude(vector))); + } + String vectorFieldName = fieldType().name(); + String vectorMagnitudeFieldName = vectorFieldName + VECTOR_MAGNITUDES_SUFFIX; + context.doc().addWithKey(vectorFieldName, new BinaryDocValuesField(vectorFieldName, new BytesRef(buffer.array()))); + context.doc() + .addWithKey( + vectorMagnitudeFieldName, + new BinaryDocValuesField(vectorMagnitudeFieldName, new BytesRef(magnitudeBuffer.array())) + ); + } + + private void checkDimensionExceeded(int index, DocumentParserContext context) { + if (index >= fieldType().dims) { + throw new IllegalArgumentException( + "The [" + + typeName() + + "] field [" + + fullPath() + + "] in doc [" + + context.documentDescription() + + "] has more dimensions " + + "than defined in the mapping [" + + fieldType().dims + + "]" + ); + } + } + + private void checkDimensionMatches(int index, DocumentParserContext context) { + if (index != fieldType().dims) { + throw new IllegalArgumentException( + "The [" + + typeName() + + "] field [" + + fullPath() + + "] in doc [" + + context.documentDescription() + + "] has a different number of dimensions " + + "[" + + index + + "] than defined in the mapping [" + + fieldType().dims + + "]" + ); + } + } + + @Override + protected void parseCreateField(DocumentParserContext context) { + throw new AssertionError("parse is implemented directly"); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + public FieldMapper.Builder getMergeBuilder() { + return new MultiDenseVectorFieldMapper.Builder(leafName(), indexCreatedVersion).init(this); + } + + @Override + protected SyntheticSourceSupport syntheticSourceSupport() { + return new SyntheticSourceSupport.Native(new MultiDenseVectorFieldMapper.DocValuesSyntheticFieldLoader()); + } + + private class DocValuesSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyntheticFieldLoader { + private BinaryDocValues values; + private boolean hasValue; + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + values = leafReader.getBinaryDocValues(fullPath()); + if (values == null) { + return null; + } + return docId -> { + hasValue = docId == values.advance(docId); + return hasValue; + }; + } + + @Override + public boolean hasValue() { + return hasValue; + } + + @Override + public void write(XContentBuilder b) throws IOException { + if (false == hasValue) { + return; + } + b.startArray(leafName()); + BytesRef ref = values.binaryValue(); + ByteBuffer byteBuffer = ByteBuffer.wrap(ref.bytes, ref.offset, ref.length).order(ByteOrder.LITTLE_ENDIAN); + assert ref.length % fieldType().elementType.getNumBytes(fieldType().dims) == 0; + int numVecs = ref.length / fieldType().elementType.getNumBytes(fieldType().dims); + for (int i = 0; i < numVecs; i++) { + b.startArray(); + int dims = fieldType().elementType == DenseVectorFieldMapper.ElementType.BIT + ? fieldType().dims / Byte.SIZE + : fieldType().dims; + for (int dim = 0; dim < dims; dim++) { + fieldType().elementType.readAndWriteValue(byteBuffer, b); + } + b.endArray(); + } + b.endArray(); + } + + @Override + public String fieldName() { + return fullPath(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorDVLeafFieldData.java new file mode 100644 index 0000000000000..cc6fb38274451 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorDVLeafFieldData.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.vectors; + +import org.apache.lucene.index.LeafReader; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; + +final class MultiVectorDVLeafFieldData implements LeafFieldData { + private final LeafReader reader; + private final String field; + private final IndexVersion indexVersion; + private final DenseVectorFieldMapper.ElementType elementType; + private final int dims; + + MultiVectorDVLeafFieldData( + LeafReader reader, + String field, + IndexVersion indexVersion, + DenseVectorFieldMapper.ElementType elementType, + int dims + ) { + this.reader = reader; + this.field = field; + this.indexVersion = indexVersion; + this.elementType = elementType; + this.dims = dims; + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + // TODO + return null; + } + + @Override + public SortedBinaryDocValues getBytesValues() { + throw new UnsupportedOperationException("String representation of doc values for multi-vector fields is not supported"); + } + + @Override + public long ramBytesUsed() { + return 0; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorIndexFieldData.java new file mode 100644 index 0000000000000..65ef492ce052b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/MultiVectorIndexFieldData.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.vectors; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +public class MultiVectorIndexFieldData implements IndexFieldData { + protected final String fieldName; + protected final ValuesSourceType valuesSourceType; + private final int dims; + private final IndexVersion indexVersion; + private final DenseVectorFieldMapper.ElementType elementType; + + public MultiVectorIndexFieldData( + String fieldName, + int dims, + ValuesSourceType valuesSourceType, + IndexVersion indexVersion, + DenseVectorFieldMapper.ElementType elementType + ) { + this.fieldName = fieldName; + this.valuesSourceType = valuesSourceType; + this.indexVersion = indexVersion; + this.elementType = elementType; + this.dims = dims; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return valuesSourceType; + } + + @Override + public MultiVectorDVLeafFieldData load(LeafReaderContext context) { + return new MultiVectorDVLeafFieldData(context.reader(), fieldName, indexVersion, elementType, dims); + } + + @Override + public MultiVectorDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + return load(context); + } + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + throw new IllegalArgumentException( + "Field [" + fieldName + "] of type [" + MultiDenseVectorFieldMapper.CONTENT_TYPE + "] doesn't support sort" + ); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("only supported on numeric fields"); + } + + public static class Builder implements IndexFieldData.Builder { + + private final String name; + private final ValuesSourceType valuesSourceType; + private final IndexVersion indexVersion; + private final int dims; + private final DenseVectorFieldMapper.ElementType elementType; + + public Builder( + String name, + ValuesSourceType valuesSourceType, + IndexVersion indexVersion, + int dims, + DenseVectorFieldMapper.ElementType elementType + ) { + this.name = name; + this.valuesSourceType = valuesSourceType; + this.indexVersion = indexVersion; + this.dims = dims; + this.elementType = elementType; + } + + @Override + public IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + return new MultiVectorIndexFieldData(name, dims, valuesSourceType, indexVersion, elementType); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java index 23d2c4b554d85..459a174d7af11 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java @@ -10,9 +10,14 @@ package org.elasticsearch.index.mapper.vectors; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.fielddata.FormattedDocValues; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; @@ -23,8 +28,12 @@ import org.elasticsearch.script.field.vectors.ByteBinaryDenseVectorDocValuesField; import org.elasticsearch.script.field.vectors.ByteKnnDenseVectorDocValuesField; import org.elasticsearch.script.field.vectors.KnnDenseVectorDocValuesField; +import org.elasticsearch.search.DocValueFormat; import java.io.IOException; +import java.util.Arrays; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; final class VectorDVLeafFieldData implements LeafFieldData { @@ -76,4 +85,115 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { } } + @Override + public FormattedDocValues getFormattedValues(DocValueFormat format) { + int dims = elementType == ElementType.BIT ? this.dims / Byte.SIZE : this.dims; + return switch (elementType) { + case BYTE, BIT -> new FormattedDocValues() { + private byte[] vector = new byte[dims]; + private ByteVectorValues byteVectorValues; // use when indexed + private BinaryDocValues binary; // use when not indexed + { + try { + if (indexed) { + byteVectorValues = reader.getByteVectorValues(field); + } else { + binary = DocValues.getBinary(reader, field); + } + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (indexed) { + if (iteratorAdvanceExact(byteVectorValues, docId) == false) { + return false; + } + vector = byteVectorValues.vectorValue(); + } else { + if (binary == null || binary.advanceExact(docId) == false) { + return false; + } + BytesRef ref = binary.binaryValue(); + System.arraycopy(ref.bytes, ref.offset, vector, 0, dims); + } + return true; + } + + @Override + public int docValueCount() { + return 1; + } + + public Object nextValue() { + Byte[] vectorValue = new Byte[dims]; + for (int i = 0; i < dims; i++) { + vectorValue[i] = vector[i]; + } + return vectorValue; + } + }; + case FLOAT -> new FormattedDocValues() { + float[] vector = new float[dims]; + private FloatVectorValues floatVectorValues; // use when indexed + private BinaryDocValues binary; // use when not indexed + { + try { + if (indexed) { + floatVectorValues = reader.getFloatVectorValues(field); + } else { + binary = DocValues.getBinary(reader, field); + } + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (indexed) { + if (iteratorAdvanceExact(floatVectorValues, docId) == false) { + return false; + } + vector = floatVectorValues.vectorValue(); + } else { + if (binary == null || binary.advanceExact(docId) == false) { + return false; + } + BytesRef ref = binary.binaryValue(); + VectorEncoderDecoder.decodeDenseVector(indexVersion, ref, vector); + } + return true; + } + + @Override + public int docValueCount() { + return 1; + } + + @Override + public Object nextValue() { + return Arrays.copyOf(vector, vector.length); + } + }; + }; + } + + private static boolean iteratorAdvanceExact(DocIdSetIterator iterator, int docId) throws IOException { + if (iterator == null) return false; + int currentDoc = iterator.docID(); + if (currentDoc == NO_MORE_DOCS || docId < currentDoc) { + return false; + } else if (docId > currentDoc) { + currentDoc = iterator.advance(docId); + if (currentDoc != docId) { + return false; + } + } + return true; + } } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 09be98630d5c4..340bff4e1c852 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -67,6 +67,7 @@ import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.MultiDenseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; @@ -210,6 +211,9 @@ public static Map getMappers(List mappe mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, DenseVectorFieldMapper.PARSER); mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, SparseVectorFieldMapper.PARSER); + if (MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()) { + mappers.put(MultiDenseVectorFieldMapper.CONTENT_TYPE, MultiDenseVectorFieldMapper.PARSER); + } for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 4efdfc66e8b5e..85c44d6162489 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -9,6 +9,10 @@ package org.elasticsearch.rest.action.search; +import org.elasticsearch.Build; +import org.elasticsearch.index.mapper.vectors.MultiDenseVectorFieldMapper; + +import java.util.HashSet; import java.util.Set; /** @@ -24,10 +28,26 @@ private SearchCapabilities() {} private static final String BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY = "bit_dense_vector_synthetic_source"; /** Support Byte and Float with Bit dot product. */ private static final String BYTE_FLOAT_BIT_DOT_PRODUCT_CAPABILITY = "byte_float_bit_dot_product"; + /** Support docvalue_fields parameter for `dense_vector` field. */ + private static final String DENSE_VECTOR_DOCVALUE_FIELDS = "dense_vector_docvalue_fields"; + /** Support kql query. */ + private static final String KQL_QUERY_SUPPORTED = "kql_query"; + /** Support multi-dense-vector field mapper. */ + private static final String MULTI_DENSE_VECTOR_FIELD_MAPPER = "multi_dense_vector_field_mapper"; - public static final Set CAPABILITIES = Set.of( - RANGE_REGEX_INTERVAL_QUERY_CAPABILITY, - BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY, - BYTE_FLOAT_BIT_DOT_PRODUCT_CAPABILITY - ); + public static final Set CAPABILITIES; + static { + HashSet capabilities = new HashSet<>(); + capabilities.add(RANGE_REGEX_INTERVAL_QUERY_CAPABILITY); + capabilities.add(BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY); + capabilities.add(BYTE_FLOAT_BIT_DOT_PRODUCT_CAPABILITY); + capabilities.add(DENSE_VECTOR_DOCVALUE_FIELDS); + if (MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()) { + capabilities.add(MULTI_DENSE_VECTOR_FIELD_MAPPER); + } + if (Build.current().isSnapshot()) { + capabilities.add(KQL_QUERY_SUPPORTED); + } + CAPABILITIES = Set.copyOf(capabilities); + } } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index f1d4f678c5fb9..bdefee988248f 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -168,6 +168,31 @@ public String toString() { } }; + DocValueFormat DENSE_VECTOR = DenseVectorDocValueFormat.INSTANCE; + + /** + * Singleton, stateless formatter, for dense vector values, no need to actually format anything + */ + class DenseVectorDocValueFormat implements DocValueFormat { + + public static final DocValueFormat INSTANCE = new DenseVectorDocValueFormat(); + + private DenseVectorDocValueFormat() {} + + @Override + public String getWriteableName() { + return "dense_vector"; + } + + @Override + public void writeTo(StreamOutput out) {} + + @Override + public String toString() { + return "dense_vector"; + } + }; + DocValueFormat BINARY = BinaryDocValueFormat.INSTANCE; /** diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 0bb914a9dbf97..fd39a95bdb75d 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -1020,6 +1020,7 @@ private void registerValueFormats() { registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP); registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW); registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY); + registerValueFormat(DocValueFormat.DENSE_VECTOR.getWriteableName(), in -> DocValueFormat.DENSE_VECTOR); registerValueFormat(DocValueFormat.UNSIGNED_LONG_SHIFTED.getWriteableName(), in -> DocValueFormat.UNSIGNED_LONG_SHIFTED); registerValueFormat(DocValueFormat.TIME_SERIES_ID.getWriteableName(), in -> DocValueFormat.TIME_SERIES_ID); registerValueFormat(TS_ROUTING_HASH_DOC_VALUE_FORMAT.getWriteableName(), in -> TS_ROUTING_HASH_DOC_VALUE_FORMAT); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java index 71de6a9bbb822..38cab1761d409 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.aggregations; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.DelayableWriteable; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -51,7 +52,12 @@ protected InternalAggregation(String name, Map metadata) { * Read from a stream. */ protected InternalAggregation(StreamInput in) throws IOException { - name = in.readString(); + final String name = in.readString(); + if (in instanceof DelayableWriteable.Deduplicator d) { + this.name = d.deduplicate(name); + } else { + this.name = name; + } metadata = in.readGenericMap(); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java index 901f0d7000542..0b9cba837583d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.tests.analysis.MockTokenizer; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -460,8 +461,8 @@ public void testExceedDefaultMaxTokenLimit() { AnalyzeAction.Request request = new AnalyzeAction.Request(); request.text(text); request.analyzer("standard"); - IllegalStateException e = expectThrows( - IllegalStateException.class, + ElasticsearchStatusException e = expectThrows( + ElasticsearchStatusException.class, () -> TransportAnalyzeAction.analyze(request, registry, null, maxTokenCount) ); assertEquals( @@ -477,8 +478,8 @@ public void testExceedDefaultMaxTokenLimit() { request2.text(text); request2.analyzer("standard"); request2.explain(true); - IllegalStateException e2 = expectThrows( - IllegalStateException.class, + ElasticsearchStatusException e2 = expectThrows( + ElasticsearchStatusException.class, () -> TransportAnalyzeAction.analyze(request2, registry, null, maxTokenCount) ); assertEquals( @@ -506,8 +507,8 @@ public void testExceedSetMaxTokenLimit() { AnalyzeAction.Request request = new AnalyzeAction.Request(); request.text(text); request.analyzer("standard"); - IllegalStateException e = expectThrows( - IllegalStateException.class, + ElasticsearchStatusException e = expectThrows( + ElasticsearchStatusException.class, () -> TransportAnalyzeAction.analyze(request, registry, null, idxMaxTokenCount) ); assertEquals( diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java index c1119ee5973f4..69872b5e4b546 100644 --- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; @@ -68,6 +69,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -77,6 +79,7 @@ import static org.elasticsearch.core.Types.forciblyCast; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.mockito.Mockito.mock; @@ -1087,6 +1090,137 @@ public void testCanMatchFilteringOnCoordinatorThatCanBeSkippedTsdb() throws Exce ); } + public void testCanMatchFilteringOnCoordinatorWithMissingShards() throws Exception { + // we'll test that we're executing _tier coordinator rewrite for indices (data stream backing or regular) without any @timestamp + // or event.ingested fields + // for both data stream backing and regular indices we'll have one index in hot and one UNASSIGNED (targeting warm though). + // the warm indices will be skipped as our queries will filter based on _tier: hot and the can match phase will not report error the + // missing index even if allow_partial_search_results is false (because the warm index would've not been part of the search anyway) + + Map indexNameToSettings = new HashMap<>(); + ClusterState state = ClusterState.EMPTY_STATE; + + String dataStreamName = randomAlphaOfLengthBetween(10, 20); + Index warmDataStreamIndex = new Index(DataStream.getDefaultBackingIndexName(dataStreamName, 1), UUIDs.base64UUID()); + indexNameToSettings.put( + warmDataStreamIndex, + settings(IndexVersion.current()).put(IndexMetadata.SETTING_INDEX_UUID, warmDataStreamIndex.getUUID()) + .put(DataTier.TIER_PREFERENCE, "data_warm,data_hot") + ); + Index hotDataStreamIndex = new Index(DataStream.getDefaultBackingIndexName(dataStreamName, 2), UUIDs.base64UUID()); + indexNameToSettings.put( + hotDataStreamIndex, + settings(IndexVersion.current()).put(IndexMetadata.SETTING_INDEX_UUID, hotDataStreamIndex.getUUID()) + .put(DataTier.TIER_PREFERENCE, "data_hot") + ); + DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, List.of(warmDataStreamIndex, hotDataStreamIndex)); + + Index warmRegularIndex = new Index("warm-index", UUIDs.base64UUID()); + indexNameToSettings.put( + warmRegularIndex, + settings(IndexVersion.current()).put(IndexMetadata.SETTING_INDEX_UUID, warmRegularIndex.getUUID()) + .put(DataTier.TIER_PREFERENCE, "data_warm,data_hot") + ); + Index hotRegularIndex = new Index("hot-index", UUIDs.base64UUID()); + indexNameToSettings.put( + hotRegularIndex, + settings(IndexVersion.current()).put(IndexMetadata.SETTING_INDEX_UUID, hotRegularIndex.getUUID()) + .put(DataTier.TIER_PREFERENCE, "data_hot") + ); + + List allIndices = new ArrayList<>(4); + allIndices.addAll(dataStream.getIndices()); + allIndices.add(warmRegularIndex); + allIndices.add(hotRegularIndex); + + List hotIndices = List.of(hotRegularIndex, hotDataStreamIndex); + List warmIndices = List.of(warmRegularIndex, warmDataStreamIndex); + + for (Index index : allIndices) { + IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(index.getName()) + .settings(indexNameToSettings.get(index)) + .numberOfShards(1) + .numberOfReplicas(0); + Metadata.Builder metadataBuilder = Metadata.builder(state.metadata()).put(indexMetadataBuilder); + state = ClusterState.builder(state).metadata(metadataBuilder).build(); + } + + ClusterState finalState = state; + CoordinatorRewriteContextProvider coordinatorRewriteContextProvider = new CoordinatorRewriteContextProvider( + parserConfig(), + mock(Client.class), + System::currentTimeMillis, + () -> finalState, + (index) -> null + ); + + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery(CoordinatorRewriteContext.TIER_FIELD_NAME, "data_hot")); + + { + // test that a search doesn't fail if the query filters out the unassigned shards + // via _tier (coordinator rewrite will eliminate the shards that don't match) + assignShardsAndExecuteCanMatchPhase( + List.of(dataStream), + List.of(hotRegularIndex, warmRegularIndex), + coordinatorRewriteContextProvider, + boolQueryBuilder, + List.of(), + null, + warmIndices, + false, + (updatedSearchShardIterators, requests) -> { + var skippedShards = updatedSearchShardIterators.stream().filter(SearchShardIterator::skip).toList(); + var nonSkippedShards = updatedSearchShardIterators.stream() + .filter(searchShardIterator -> searchShardIterator.skip() == false) + .toList(); + + boolean allSkippedShardAreFromWarmIndices = skippedShards.stream() + .allMatch(shardIterator -> warmIndices.contains(shardIterator.shardId().getIndex())); + assertThat(allSkippedShardAreFromWarmIndices, equalTo(true)); + boolean allNonSkippedShardAreHotIndices = nonSkippedShards.stream() + .allMatch(shardIterator -> hotIndices.contains(shardIterator.shardId().getIndex())); + assertThat(allNonSkippedShardAreHotIndices, equalTo(true)); + boolean allRequestMadeToHotIndices = requests.stream() + .allMatch(request -> hotIndices.contains(request.shardId().getIndex())); + assertThat(allRequestMadeToHotIndices, equalTo(true)); + } + ); + } + + { + // test that a search does fail if the query does NOT filter ALL the + // unassigned shards + CountDownLatch latch = new CountDownLatch(1); + Tuple> canMatchPhaseAndRequests = getCanMatchPhaseAndRequests( + List.of(dataStream), + List.of(hotRegularIndex, warmRegularIndex), + coordinatorRewriteContextProvider, + boolQueryBuilder, + List.of(), + null, + List.of(hotRegularIndex, warmRegularIndex, warmDataStreamIndex), + false, + new ActionListener<>() { + @Override + public void onResponse(GroupShardsIterator searchShardIterators) { + fail(null, "unexpected success with result [%s] while expecting to handle failure with [%s]", searchShardIterators); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + assertThat(e, instanceOf(SearchPhaseExecutionException.class)); + latch.countDown(); + } + } + ); + + canMatchPhaseAndRequests.v1().start(); + latch.await(10, TimeUnit.SECONDS); + } + } + private void assertAllShardsAreQueried(List updatedSearchShardIterators, List requests) { int skippedShards = (int) updatedSearchShardIterators.stream().filter(SearchShardIterator::skip).count(); @@ -1111,6 +1245,69 @@ private void assignShardsAndExecuteCanMatchPhase( SuggestBuilder suggest, BiConsumer, List> canMatchResultsConsumer ) throws Exception { + assignShardsAndExecuteCanMatchPhase( + dataStreams, + regularIndices, + contextProvider, + query, + aggregations, + suggest, + List.of(), + true, + canMatchResultsConsumer + ); + } + + private void assignShardsAndExecuteCanMatchPhase( + List dataStreams, + List regularIndices, + CoordinatorRewriteContextProvider contextProvider, + QueryBuilder query, + List aggregations, + SuggestBuilder suggest, + List unassignedIndices, + boolean allowPartialResults, + BiConsumer, List> canMatchResultsConsumer + ) throws Exception { + AtomicReference> result = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + Tuple> canMatchAndShardRequests = getCanMatchPhaseAndRequests( + dataStreams, + regularIndices, + contextProvider, + query, + aggregations, + suggest, + unassignedIndices, + allowPartialResults, + ActionTestUtils.assertNoFailureListener(iter -> { + result.set(iter); + latch.countDown(); + }) + ); + + canMatchAndShardRequests.v1().start(); + latch.await(); + + List updatedSearchShardIterators = new ArrayList<>(); + for (SearchShardIterator updatedSearchShardIterator : result.get()) { + updatedSearchShardIterators.add(updatedSearchShardIterator); + } + + canMatchResultsConsumer.accept(updatedSearchShardIterators, canMatchAndShardRequests.v2()); + } + + private Tuple> getCanMatchPhaseAndRequests( + List dataStreams, + List regularIndices, + CoordinatorRewriteContextProvider contextProvider, + QueryBuilder query, + List aggregations, + SuggestBuilder suggest, + List unassignedIndices, + boolean allowPartialResults, + ActionListener> canMatchActionListener + ) { Map lookup = new ConcurrentHashMap<>(); DiscoveryNode primaryNode = DiscoveryNodeUtils.create("node_1"); DiscoveryNode replicaNode = DiscoveryNodeUtils.create("node_2"); @@ -1136,23 +1333,31 @@ private void assignShardsAndExecuteCanMatchPhase( // and none is assigned, the phase is considered as failed meaning that the next phase won't be executed boolean withAssignedPrimaries = randomBoolean() || atLeastOnePrimaryAssigned == false; int numShards = randomIntBetween(1, 6); - originalShardIters.addAll( - getShardsIter(dataStreamIndex, originalIndices, numShards, false, withAssignedPrimaries ? primaryNode : null, null) - ); - atLeastOnePrimaryAssigned |= withAssignedPrimaries; + if (unassignedIndices.contains(dataStreamIndex)) { + originalShardIters.addAll(getShardsIter(dataStreamIndex, originalIndices, numShards, false, null, null)); + } else { + originalShardIters.addAll( + getShardsIter(dataStreamIndex, originalIndices, numShards, false, withAssignedPrimaries ? primaryNode : null, null) + ); + atLeastOnePrimaryAssigned |= withAssignedPrimaries; + } } } for (Index regularIndex : regularIndices) { - originalShardIters.addAll( - getShardsIter(regularIndex, originalIndices, randomIntBetween(1, 6), randomBoolean(), primaryNode, replicaNode) - ); + if (unassignedIndices.contains(regularIndex)) { + originalShardIters.addAll(getShardsIter(regularIndex, originalIndices, randomIntBetween(1, 6), false, null, null)); + } else { + originalShardIters.addAll( + getShardsIter(regularIndex, originalIndices, randomIntBetween(1, 6), randomBoolean(), primaryNode, replicaNode) + ); + } } GroupShardsIterator shardsIter = GroupShardsIterator.sortAndCreate(originalShardIters); final SearchRequest searchRequest = new SearchRequest(); searchRequest.indices(indices); - searchRequest.allowPartialSearchResults(true); + searchRequest.allowPartialSearchResults(allowPartialResults); final AliasFilter aliasFilter; if (aggregations.isEmpty() == false || randomBoolean()) { @@ -1212,35 +1417,24 @@ public void sendCanMatch( ); AtomicReference> result = new AtomicReference<>(); - CountDownLatch latch = new CountDownLatch(1); - CanMatchPreFilterSearchPhase canMatchPhase = new CanMatchPreFilterSearchPhase( - logger, - searchTransportService, - (clusterAlias, node) -> lookup.get(node), - aliasFilters, - Collections.emptyMap(), - threadPool.executor(ThreadPool.Names.SEARCH_COORDINATION), - searchRequest, - shardsIter, - timeProvider, - null, - true, - contextProvider, - ActionTestUtils.assertNoFailureListener(iter -> { - result.set(iter); - latch.countDown(); - }) + return new Tuple<>( + new CanMatchPreFilterSearchPhase( + logger, + searchTransportService, + (clusterAlias, node) -> lookup.get(node), + aliasFilters, + Collections.emptyMap(), + threadPool.executor(ThreadPool.Names.SEARCH_COORDINATION), + searchRequest, + shardsIter, + timeProvider, + null, + true, + contextProvider, + canMatchActionListener + ), + requests ); - - canMatchPhase.start(); - latch.await(); - - List updatedSearchShardIterators = new ArrayList<>(); - for (SearchShardIterator updatedSearchShardIterator : result.get()) { - updatedSearchShardIterators.add(updatedSearchShardIterator); - } - - canMatchResultsConsumer.accept(updatedSearchShardIterators, requests); } static class StaticCoordinatorRewriteContextProviderBuilder { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 6433cf2f1c0d4..9e819f38eae6e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.DenseVectorFieldType; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.VectorSimilarity; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.vectors.DenseVectorQuery; import org.elasticsearch.search.vectors.VectorData; @@ -134,9 +135,9 @@ public void testFielddataBuilder() { public void testDocValueFormat() { DenseVectorFieldType fft = createFloatFieldType(); - expectThrows(IllegalArgumentException.class, () -> fft.docValueFormat(null, null)); + assertEquals(DocValueFormat.DENSE_VECTOR, fft.docValueFormat(null, null)); DenseVectorFieldType bft = createByteFieldType(); - expectThrows(IllegalArgumentException.class, () -> bft.docValueFormat(null, null)); + assertEquals(DocValueFormat.DENSE_VECTOR, bft.docValueFormat(null, null)); } public void testFetchSourceValue() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapperTests.java new file mode 100644 index 0000000000000..6a890328732ca --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldMapperTests.java @@ -0,0 +1,506 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.vectors; + +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.FieldExistsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentParsingException; +import org.elasticsearch.index.mapper.LuceneDocument; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperTestCase; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceProvider; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.junit.AssumptionViolatedException; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MultiDenseVectorFieldMapperTests extends MapperTestCase { + + @BeforeClass + public static void setup() { + assumeTrue("Requires multi-dense vector support", MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()); + } + + private final ElementType elementType; + private final int dims; + + public MultiDenseVectorFieldMapperTests() { + this.elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT, ElementType.BIT); + this.dims = ElementType.BIT == elementType ? 4 * Byte.SIZE : 4; + } + + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + indexMapping(b, IndexVersion.current()); + } + + @Override + protected void minimalMapping(XContentBuilder b, IndexVersion indexVersion) throws IOException { + indexMapping(b, indexVersion); + } + + private void indexMapping(XContentBuilder b, IndexVersion indexVersion) throws IOException { + b.field("type", "multi_dense_vector").field("dims", dims); + if (elementType != ElementType.FLOAT) { + b.field("element_type", elementType.toString()); + } + } + + @Override + protected Object getSampleValueForDocument() { + int numVectors = randomIntBetween(1, 16); + return Stream.generate( + () -> elementType == ElementType.FLOAT ? List.of(0.5, 0.5, 0.5, 0.5) : List.of((byte) 1, (byte) 1, (byte) 1, (byte) 1) + ).limit(numVectors).toList(); + } + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException { + checker.registerConflictCheck( + "dims", + fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims)), + fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims + 8)) + ); + checker.registerConflictCheck( + "element_type", + fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims).field("element_type", "byte")), + fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims).field("element_type", "float")) + ); + checker.registerConflictCheck( + "element_type", + fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims).field("element_type", "float")), + fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims * 8).field("element_type", "bit")) + ); + checker.registerConflictCheck( + "element_type", + fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims).field("element_type", "byte")), + fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", dims * 8).field("element_type", "bit")) + ); + } + + @Override + protected boolean supportsStoredFields() { + return false; + } + + @Override + protected boolean supportsIgnoreMalformed() { + return false; + } + + @Override + protected void assertSearchable(MappedFieldType fieldType) { + assertThat(fieldType, instanceOf(MultiDenseVectorFieldMapper.MultiDenseVectorFieldType.class)); + assertFalse(fieldType.isIndexed()); + assertFalse(fieldType.isSearchable()); + } + + protected void assertExistsQuery(MappedFieldType fieldType, Query query, LuceneDocument fields) { + assertThat(query, instanceOf(FieldExistsQuery.class)); + FieldExistsQuery existsQuery = (FieldExistsQuery) query; + assertEquals("field", existsQuery.getField()); + assertNoFieldNamesField(fields); + } + + // We override this because dense vectors are the only field type that are not aggregatable but + // that do provide fielddata. TODO: resolve this inconsistency! + @Override + public void testAggregatableConsistency() {} + + public void testDims() { + { + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { + b.field("type", "multi_dense_vector"); + b.field("dims", 0); + }))); + assertThat( + e.getMessage(), + equalTo("Failed to parse mapping: " + "The number of dimensions should be in the range [1, 4096] but was [0]") + ); + } + // test max limit for non-indexed vectors + { + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { + b.field("type", "multi_dense_vector"); + b.field("dims", 5000); + }))); + assertThat( + e.getMessage(), + equalTo("Failed to parse mapping: " + "The number of dimensions should be in the range [1, 4096] but was [5000]") + ); + } + } + + public void testMergeDims() throws IOException { + XContentBuilder mapping = mapping(b -> { + b.startObject("field"); + b.field("type", "multi_dense_vector"); + b.endObject(); + }); + MapperService mapperService = createMapperService(mapping); + + mapping = mapping(b -> { + b.startObject("field"); + b.field("type", "multi_dense_vector").field("dims", dims); + b.endObject(); + }); + merge(mapperService, mapping); + assertEquals( + XContentHelper.convertToMap(BytesReference.bytes(mapping), false, mapping.contentType()).v2(), + XContentHelper.convertToMap(mapperService.documentMapper().mappingSource().uncompressed(), false, mapping.contentType()).v2() + ); + } + + public void testLargeDimsBit() throws IOException { + createMapperService(fieldMapping(b -> { + b.field("type", "multi_dense_vector"); + b.field("dims", 1024 * Byte.SIZE); + b.field("element_type", ElementType.BIT.toString()); + })); + } + + public void testNonIndexedVector() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", 3))); + + float[][] validVectors = { { -12.1f, 100.7f, -4 }, { 42f, .05f, -1f } }; + double[] dotProduct = new double[2]; + int vecId = 0; + for (float[] vector : validVectors) { + for (float value : vector) { + dotProduct[vecId] += value * value; + } + vecId++; + } + ParsedDocument doc1 = mapper.parse(source(b -> { + b.startArray("field"); + for (float[] vector : validVectors) { + b.startArray(); + for (float value : vector) { + b.value(value); + } + b.endArray(); + } + b.endArray(); + })); + + List fields = doc1.rootDoc().getFields("field"); + assertEquals(1, fields.size()); + assertThat(fields.get(0), instanceOf(BinaryDocValuesField.class)); + // assert that after decoding the indexed value is equal to expected + BytesRef vectorBR = fields.get(0).binaryValue(); + assertEquals(ElementType.FLOAT.getNumBytes(validVectors[0].length) * validVectors.length, vectorBR.length); + float[][] decodedValues = new float[validVectors.length][]; + for (int i = 0; i < validVectors.length; i++) { + decodedValues[i] = new float[validVectors[i].length]; + FloatBuffer fb = ByteBuffer.wrap(vectorBR.bytes, i * Float.BYTES * validVectors[i].length, Float.BYTES * validVectors[i].length) + .order(ByteOrder.LITTLE_ENDIAN) + .asFloatBuffer(); + fb.get(decodedValues[i]); + } + List magFields = doc1.rootDoc().getFields("field" + MultiDenseVectorFieldMapper.VECTOR_MAGNITUDES_SUFFIX); + assertEquals(1, magFields.size()); + assertThat(magFields.get(0), instanceOf(BinaryDocValuesField.class)); + BytesRef magBR = magFields.get(0).binaryValue(); + assertEquals(Float.BYTES * validVectors.length, magBR.length); + FloatBuffer fb = ByteBuffer.wrap(magBR.bytes, magBR.offset, magBR.length).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer(); + for (int i = 0; i < validVectors.length; i++) { + assertEquals((float) Math.sqrt(dotProduct[i]), fb.get(), 0.001f); + } + for (int i = 0; i < validVectors.length; i++) { + assertArrayEquals("Decoded dense vector values is not equal to the indexed one.", validVectors[i], decodedValues[i], 0.001f); + } + } + + public void testPoorlyIndexedVector() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", 3))); + + float[][] validVectors = { { -12.1f, 100.7f, -4 }, { 42f, .05f, -1f } }; + double[] dotProduct = new double[2]; + int vecId = 0; + for (float[] vector : validVectors) { + for (float value : vector) { + dotProduct[vecId] += value * value; + } + vecId++; + } + expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> { + b.startArray("field"); + b.startArray(); // double nested array should fail + for (float[] vector : validVectors) { + b.startArray(); + for (float value : vector) { + b.value(value); + } + b.endArray(); + } + b.endArray(); + b.endArray(); + }))); + } + + public void testInvalidParameters() { + + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> createDocumentMapper( + fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", 3).field("element_type", "foo")) + ) + ); + assertThat(e.getMessage(), containsString("invalid element_type [foo]; available types are ")); + e = expectThrows( + MapperParsingException.class, + () -> createDocumentMapper( + fieldMapping(b -> b.field("type", "multi_dense_vector").field("dims", 3).startObject("foo").endObject()) + ) + ); + assertThat( + e.getMessage(), + containsString("Failed to parse mapping: unknown parameter [foo] on mapper [field] of type [multi_dense_vector]") + ); + } + + public void testDocumentsWithIncorrectDims() throws Exception { + int dims = 3; + XContentBuilder fieldMapping = fieldMapping(b -> { + b.field("type", "multi_dense_vector"); + b.field("dims", dims); + }); + + DocumentMapper mapper = createDocumentMapper(fieldMapping); + + // test that error is thrown when a document has number of dims more than defined in the mapping + float[][] invalidVector = new float[4][dims + 1]; + DocumentParsingException e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> { + b.startArray("field"); + for (float[] vector : invalidVector) { + b.startArray(); + for (float value : vector) { + b.value(value); + } + b.endArray(); + } + b.endArray(); + }))); + assertThat(e.getCause().getMessage(), containsString("has more dimensions than defined in the mapping [3]")); + + // test that error is thrown when a document has number of dims less than defined in the mapping + float[][] invalidVector2 = new float[4][dims - 1]; + DocumentParsingException e2 = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> { + b.startArray("field"); + for (float[] vector : invalidVector2) { + b.startArray(); + for (float value : vector) { + b.value(value); + } + b.endArray(); + } + b.endArray(); + }))); + assertThat(e2.getCause().getMessage(), containsString("has a different number of dimensions [2] than defined in the mapping [3]")); + // test that error is thrown when some of the vectors have correct number of dims, but others do not + DocumentParsingException e3 = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> { + b.startArray("field"); + for (float[] vector : new float[4][dims]) { + b.startArray(); + for (float value : vector) { + b.value(value); + } + b.endArray(); + } + for (float[] vector : invalidVector2) { + b.startArray(); + for (float value : vector) { + b.value(value); + } + b.endArray(); + } + b.endArray(); + }))); + assertThat(e3.getCause().getMessage(), containsString("has a different number of dimensions [2] than defined in the mapping [3]")); + } + + @Override + protected void assertFetchMany(MapperService mapperService, String field, Object value, String format, int count) throws IOException { + assumeFalse("Dense vectors currently don't support multiple values in the same field", false); + } + + /** + * Dense vectors don't support doc values or string representation (for doc value parser/fetching). + * We may eventually support that, but until then, we only verify that the parsing and fields fetching matches the provided value object + */ + @Override + protected void assertFetch(MapperService mapperService, String field, Object value, String format) throws IOException { + MappedFieldType ft = mapperService.fieldType(field); + MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; + SourceToParse source = source(b -> b.field(ft.name(), value)); + SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.isSourceEnabled()).thenReturn(true); + when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); + when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { + throw new UnsupportedOperationException(); + }, fdt)); + ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); + ParsedDocument doc = mapperService.documentMapper().parse(source); + withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { + Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + nativeFetcher.setNextReader(ir.leaves().get(0)); + List fromNative = nativeFetcher.fetchValues(s, 0, new ArrayList<>()); + MultiDenseVectorFieldMapper.MultiDenseVectorFieldType denseVectorFieldType = + (MultiDenseVectorFieldMapper.MultiDenseVectorFieldType) ft; + switch (denseVectorFieldType.getElementType()) { + case BYTE -> assumeFalse("byte element type testing not currently added", false); + case FLOAT -> { + List fetchedFloatsList = new ArrayList<>(); + for (var f : fromNative) { + float[] fetchedFloats = new float[denseVectorFieldType.getVectorDimensions()]; + assert f instanceof List; + List vector = (List) f; + int i = 0; + for (Object v : vector) { + assert v instanceof Number; + fetchedFloats[i++] = ((Number) v).floatValue(); + } + fetchedFloatsList.add(fetchedFloats); + } + float[][] fetchedFloats = fetchedFloatsList.toArray(new float[0][]); + assertThat("fetching " + value, fetchedFloats, equalTo(value)); + } + } + }); + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", "multi_dense_vector").field("dims", randomIntBetween(2, 4096)).field("element_type", "float"); + } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + MultiDenseVectorFieldMapper.MultiDenseVectorFieldType vectorFieldType = (MultiDenseVectorFieldMapper.MultiDenseVectorFieldType) ft; + int numVectors = randomIntBetween(1, 16); + return switch (vectorFieldType.getElementType()) { + case BYTE -> { + byte[][] vectors = new byte[numVectors][vectorFieldType.getVectorDimensions()]; + for (int i = 0; i < numVectors; i++) { + vectors[i] = randomByteArrayOfLength(vectorFieldType.getVectorDimensions()); + } + yield vectors; + } + case FLOAT -> { + float[][] vectors = new float[numVectors][vectorFieldType.getVectorDimensions()]; + for (int i = 0; i < numVectors; i++) { + for (int j = 0; j < vectorFieldType.getVectorDimensions(); j++) { + vectors[i][j] = randomFloat(); + } + } + yield vectors; + } + case BIT -> { + byte[][] vectors = new byte[numVectors][vectorFieldType.getVectorDimensions() / 8]; + for (int i = 0; i < numVectors; i++) { + vectors[i] = randomByteArrayOfLength(vectorFieldType.getVectorDimensions() / 8); + } + yield vectors; + } + }; + } + + public void testCannotBeUsedInMultifields() { + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { + b.field("type", "keyword"); + b.startObject("fields"); + b.startObject("vectors"); + minimalMapping(b); + b.endObject(); + b.endObject(); + }))); + assertThat(e.getMessage(), containsString("Field [vectors] of type [multi_dense_vector] can't be used in multifields")); + } + + @Override + protected IngestScriptSupport ingestScriptSupport() { + throw new AssumptionViolatedException("not supported"); + } + + @Override + protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { + return new DenseVectorSyntheticSourceSupport(); + } + + @Override + protected boolean supportsEmptyInputArray() { + return false; + } + + private static class DenseVectorSyntheticSourceSupport implements SyntheticSourceSupport { + private final int dims = between(5, 1000); + private final int numVecs = between(1, 16); + private final ElementType elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT, ElementType.BIT); + + @Override + public SyntheticSourceExample example(int maxValues) { + Object value = switch (elementType) { + case BYTE, BIT: + yield randomList(numVecs, numVecs, () -> randomList(dims, dims, ESTestCase::randomByte)); + case FLOAT: + yield randomList(numVecs, numVecs, () -> randomList(dims, dims, ESTestCase::randomFloat)); + }; + return new SyntheticSourceExample(value, value, this::mapping); + } + + private void mapping(XContentBuilder b) throws IOException { + b.field("type", "multi_dense_vector"); + if (elementType == ElementType.BYTE || elementType == ElementType.BIT || randomBoolean()) { + b.field("element_type", elementType.toString()); + } + b.field("dims", elementType == ElementType.BIT ? dims * Byte.SIZE : dims); + } + + @Override + public List invalidExample() { + return List.of(); + } + } + + @Override + public void testSyntheticSourceKeepArrays() { + // The mapper expects to parse an array of values by default, it's not compatible with array of arrays. + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldTypeTests.java new file mode 100644 index 0000000000000..14cc63e31fa27 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/MultiDenseVectorFieldTypeTests.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.vectors; + +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.fielddata.FieldDataContext; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.vectors.MultiDenseVectorFieldMapper.MultiDenseVectorFieldType; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS; + +public class MultiDenseVectorFieldTypeTests extends FieldTypeTestCase { + + @BeforeClass + public static void setup() { + assumeTrue("Requires multi-dense vector support", MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()); + } + + private MultiDenseVectorFieldType createFloatFieldType() { + return new MultiDenseVectorFieldType( + "f", + DenseVectorFieldMapper.ElementType.FLOAT, + BBQ_MIN_DIMS, + IndexVersion.current(), + Collections.emptyMap() + ); + } + + private MultiDenseVectorFieldType createByteFieldType() { + return new MultiDenseVectorFieldType( + "f", + DenseVectorFieldMapper.ElementType.BYTE, + 5, + IndexVersion.current(), + Collections.emptyMap() + ); + } + + public void testHasDocValues() { + MultiDenseVectorFieldType fft = createFloatFieldType(); + assertTrue(fft.hasDocValues()); + MultiDenseVectorFieldType bft = createByteFieldType(); + assertTrue(bft.hasDocValues()); + } + + public void testIsIndexed() { + MultiDenseVectorFieldType fft = createFloatFieldType(); + assertFalse(fft.isIndexed()); + MultiDenseVectorFieldType bft = createByteFieldType(); + assertFalse(bft.isIndexed()); + } + + public void testIsSearchable() { + MultiDenseVectorFieldType fft = createFloatFieldType(); + assertFalse(fft.isSearchable()); + MultiDenseVectorFieldType bft = createByteFieldType(); + assertFalse(bft.isSearchable()); + } + + public void testIsAggregatable() { + MultiDenseVectorFieldType fft = createFloatFieldType(); + assertFalse(fft.isAggregatable()); + MultiDenseVectorFieldType bft = createByteFieldType(); + assertFalse(bft.isAggregatable()); + } + + public void testFielddataBuilder() { + MultiDenseVectorFieldType fft = createFloatFieldType(); + FieldDataContext fdc = new FieldDataContext("test", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + assertNotNull(fft.fielddataBuilder(fdc)); + + MultiDenseVectorFieldType bft = createByteFieldType(); + FieldDataContext bdc = new FieldDataContext("test", null, () -> null, Set::of, MappedFieldType.FielddataOperation.SCRIPT); + assertNotNull(bft.fielddataBuilder(bdc)); + } + + public void testDocValueFormat() { + MultiDenseVectorFieldType fft = createFloatFieldType(); + expectThrows(IllegalArgumentException.class, () -> fft.docValueFormat(null, null)); + MultiDenseVectorFieldType bft = createByteFieldType(); + expectThrows(IllegalArgumentException.class, () -> bft.docValueFormat(null, null)); + } + + public void testFetchSourceValue() throws IOException { + MultiDenseVectorFieldType fft = createFloatFieldType(); + List> vector = List.of(List.of(0.0, 1.0, 2.0, 3.0, 4.0, 6.0)); + assertEquals(vector, fetchSourceValue(fft, vector)); + MultiDenseVectorFieldType bft = createByteFieldType(); + assertEquals(vector, fetchSourceValue(bft, vector)); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index d35d5282238ee..3c45a8aa5dd45 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -109,6 +109,7 @@ import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.MultiDenseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; @@ -199,6 +200,7 @@ public abstract class AggregatorTestCase extends ESTestCase { private static final List TYPE_TEST_BLACKLIST = List.of( ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects DenseVectorFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors + MultiDenseVectorFieldMapper.CONTENT_TYPE, // Cannot aggregate dense vectors SparseVectorFieldMapper.CONTENT_TYPE, // Sparse vectors are no longer supported NestedObjectMapper.CONTENT_TYPE, // TODO support for nested diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 2b0a672d03602..e9978ab1aca6a 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -207,5 +207,6 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task -> task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry) non-snapshot version", "The number of functions is constantly increasing") task.skipTest("esql/80_text/reverse text", "The output type changed from TEXT to KEYWORD.") task.skipTest("esql/80_text/values function", "The output type changed from TEXT to KEYWORD.") + task.skipTest("privileges/11_builtin/Test get builtin privileges" ,"unnecessary to test compatibility") }) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java index 916bd3c62a598..9bbe41b4797fe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java @@ -36,7 +36,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { private static final String ELASTICEARCH_VERSION_DEPRECATION_MESSAGE = - "the [elasticsearch_version] field of an enrich policy has no effect and will be removed in Elasticsearch 9.0"; + "the [elasticsearch_version] field of an enrich policy has no effect and will be removed in a future version of Elasticsearch"; private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(EnrichPolicy.class); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java index de351cd59c690..763ab6ccb9886 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java @@ -115,7 +115,7 @@ public boolean hasRemoteIndicesPrivileges() { } public boolean hasRemoteClusterPrivileges() { - return remoteClusterPermissions.hasPrivileges(); + return remoteClusterPermissions.hasAnyPrivileges(); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java index 04dda75692208..c2f40a3e393b9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings; import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.InternalUsers; @@ -76,6 +77,7 @@ import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.FALLBACK_REALM_NAME; import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.FALLBACK_REALM_TYPE; import static org.elasticsearch.xpack.core.security.authc.RealmDomain.REALM_DOMAIN_PARSER; +import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.Fields.REMOTE_CLUSTER; import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS; /** @@ -233,8 +235,8 @@ public Authentication maybeRewriteForOlderVersion(TransportVersion olderVersion) + "]" ); } - final Map newMetadata = maybeRewriteMetadata(olderVersion, this); + final Authentication newAuthentication; if (isRunAs()) { // The lookup user for run-as currently doesn't have authentication metadata associated with them because @@ -272,12 +274,23 @@ public Authentication maybeRewriteForOlderVersion(TransportVersion olderVersion) } private static Map maybeRewriteMetadata(TransportVersion olderVersion, Authentication authentication) { - if (authentication.isAuthenticatedAsApiKey()) { - return maybeRewriteMetadataForApiKeyRoleDescriptors(olderVersion, authentication); - } else if (authentication.isCrossClusterAccess()) { - return maybeRewriteMetadataForCrossClusterAccessAuthentication(olderVersion, authentication); - } else { - return authentication.getAuthenticatingSubject().getMetadata(); + try { + if (authentication.isAuthenticatedAsApiKey()) { + return maybeRewriteMetadataForApiKeyRoleDescriptors(olderVersion, authentication); + } else if (authentication.isCrossClusterAccess()) { + return maybeRewriteMetadataForCrossClusterAccessAuthentication(olderVersion, authentication); + } else { + return authentication.getAuthenticatingSubject().getMetadata(); + } + } catch (Exception e) { + // CCS workflows may swallow the exception message making this difficult to troubleshoot, so we explicitly log and re-throw + // here. It may result in duplicate logs, so we only log the message at warn level. + if (logger.isDebugEnabled()) { + logger.debug("Un-expected exception thrown while rewriting metadata. This is likely a bug.", e); + } else { + logger.warn("Un-expected exception thrown while rewriting metadata. This is likely a bug [" + e.getMessage() + "]"); + } + throw e; } } @@ -1323,6 +1336,7 @@ private static Map maybeRewriteMetadataForApiKeyRoleDescriptors( if (authentication.getEffectiveSubject().getTransportVersion().onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS) && streamVersion.before(ROLE_REMOTE_CLUSTER_PRIVS)) { + // the authentication understands the remote_cluster field but the stream does not metadata = new HashMap<>(metadata); metadata.put( AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY, @@ -1336,7 +1350,26 @@ private static Map maybeRewriteMetadataForApiKeyRoleDescriptors( (BytesReference) metadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY) ) ); - } + } else if (authentication.getEffectiveSubject().getTransportVersion().onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS) + && streamVersion.onOrAfter(ROLE_REMOTE_CLUSTER_PRIVS)) { + // both the authentication object and the stream understand the remote_cluster field + // check each individual permission and remove as needed + metadata = new HashMap<>(metadata); + metadata.put( + AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY, + maybeRemoveRemoteClusterPrivilegesFromRoleDescriptors( + (BytesReference) metadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY), + streamVersion + ) + ); + metadata.put( + AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, + maybeRemoveRemoteClusterPrivilegesFromRoleDescriptors( + (BytesReference) metadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY), + streamVersion + ) + ); + } if (authentication.getEffectiveSubject().getTransportVersion().onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES) && streamVersion.before(VERSION_API_KEY_ROLES_AS_BYTES)) { @@ -1417,7 +1450,7 @@ private static BytesReference convertRoleDescriptorsMapToBytes(Map roleDescriptorsMap = convertRoleDescriptorsBytesToMap(roleDescriptorsBytes); + final Map roleDescriptorsMapMutated = new HashMap<>(roleDescriptorsMap); + final AtomicBoolean modified = new AtomicBoolean(false); + roleDescriptorsMap.forEach((key, value) -> { + if (value instanceof Map) { + Map roleDescriptor = (Map) value; + roleDescriptor.forEach((innerKey, innerValue) -> { + // example: remote_cluster=[{privileges=[monitor_enrich, monitor_stats] + if (REMOTE_CLUSTER.getPreferredName().equals(innerKey)) { + assert innerValue instanceof List; + RemoteClusterPermissions discoveredRemoteClusterPermission = new RemoteClusterPermissions( + (List>>) innerValue + ); + RemoteClusterPermissions mutated = discoveredRemoteClusterPermission.removeUnsupportedPrivileges(outboundVersion); + if (mutated.equals(discoveredRemoteClusterPermission) == false) { + // swap out the old value with the new value + modified.set(true); + Map remoteClusterMap = new HashMap<>((Map) roleDescriptorsMapMutated.get(key)); + if (mutated.hasAnyPrivileges()) { + // has at least one group with privileges + remoteClusterMap.put(innerKey, mutated.toMap()); + } else { + // has no groups with privileges + remoteClusterMap.remove(innerKey); + } + roleDescriptorsMapMutated.put(key, remoteClusterMap); + } + } + }); + } + }); + if (modified.get()) { + logger.debug( + "mutated role descriptors. Changed from {} to {} for outbound version {}", + roleDescriptorsMap, + roleDescriptorsMapMutated, + outboundVersion + ); + return convertRoleDescriptorsMapToBytes(roleDescriptorsMapMutated); + } else { + // No need to serialize if we did not change anything. + logger.trace("no change to role descriptors {} for outbound version {}", roleDescriptorsMap, outboundVersion); + return roleDescriptorsBytes; + } + } + static boolean equivalentRealms(String name1, String type1, String name2, String type2) { if (false == type1.equals(type2)) { return false; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java index 8d069caf0496f..9f5aaa8562a88 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.core.security.authz; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.TransportVersion; @@ -62,6 +64,7 @@ public class RoleDescriptor implements ToXContentObject, Writeable { public static final TransportVersion SECURITY_ROLE_DESCRIPTION = TransportVersions.V_8_15_0; public static final String ROLE_TYPE = "role"; + private static final Logger logger = LogManager.getLogger(RoleDescriptor.class); private final String name; private final String[] clusterPrivileges; @@ -191,7 +194,7 @@ public RoleDescriptor( ? Collections.unmodifiableMap(transientMetadata) : Collections.singletonMap("enabled", true); this.remoteIndicesPrivileges = remoteIndicesPrivileges != null ? remoteIndicesPrivileges : RemoteIndicesPrivileges.NONE; - this.remoteClusterPermissions = remoteClusterPermissions != null && remoteClusterPermissions.hasPrivileges() + this.remoteClusterPermissions = remoteClusterPermissions != null && remoteClusterPermissions.hasAnyPrivileges() ? remoteClusterPermissions : RemoteClusterPermissions.NONE; this.restriction = restriction != null ? restriction : Restriction.NONE; @@ -263,7 +266,7 @@ public boolean hasRemoteIndicesPrivileges() { } public boolean hasRemoteClusterPermissions() { - return remoteClusterPermissions.hasPrivileges(); + return remoteClusterPermissions.hasAnyPrivileges(); } public RemoteClusterPermissions getRemoteClusterPermissions() { @@ -830,25 +833,32 @@ private static RemoteClusterPermissions parseRemoteCluster(final String roleName currentFieldName = parser.currentName(); } else if (Fields.PRIVILEGES.match(currentFieldName, parser.getDeprecationHandler())) { privileges = readStringArray(roleName, parser, false); - if (privileges.length != 1 - || RemoteClusterPermissions.getSupportedRemoteClusterPermissions() - .contains(privileges[0].trim().toLowerCase(Locale.ROOT)) == false) { - throw new ElasticsearchParseException( - "failed to parse remote_cluster for role [{}]. " - + RemoteClusterPermissions.getSupportedRemoteClusterPermissions() - + " is the only value allowed for [{}] within [remote_cluster]", + if (Arrays.stream(privileges) + .map(s -> s.toLowerCase(Locale.ROOT).trim()) + .allMatch(RemoteClusterPermissions.getSupportedRemoteClusterPermissions()::contains) == false) { + final String message = String.format( + Locale.ROOT, + "failed to parse remote_cluster for role [%s]. " + + "%s are the only values allowed for [%s] within [remote_cluster]. Found %s", roleName, - currentFieldName + RemoteClusterPermissions.getSupportedRemoteClusterPermissions(), + currentFieldName, + Arrays.toString(privileges) ); + logger.info(message); + throw new ElasticsearchParseException(message); } } else if (Fields.CLUSTERS.match(currentFieldName, parser.getDeprecationHandler())) { clusters = readStringArray(roleName, parser, false); } else { - throw new ElasticsearchParseException( - "failed to parse remote_cluster for role [{}]. unexpected field [{}]", + final String message = String.format( + Locale.ROOT, + "failed to parse remote_cluster for role [%s]. unexpected field [%s]", roleName, currentFieldName ); + logger.info(message); + throw new ElasticsearchParseException(message); } } if (privileges != null && clusters == null) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionGroup.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionGroup.java index 1c34a7829fcbb..ec245fae28612 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionGroup.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionGroup.java @@ -13,11 +13,15 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.support.StringMatcher; import java.io.IOException; import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.Fields.CLUSTERS; +import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.Fields.PRIVILEGES; /** * Represents a group of permissions for a remote cluster. For example: @@ -41,6 +45,14 @@ public RemoteClusterPermissionGroup(StreamInput in) throws IOException { remoteClusterAliasMatcher = StringMatcher.of(remoteClusterAliases); } + public RemoteClusterPermissionGroup(Map> remoteClusterGroup) { + assert remoteClusterGroup.get(PRIVILEGES.getPreferredName()) != null : "privileges must be non-null"; + assert remoteClusterGroup.get(CLUSTERS.getPreferredName()) != null : "clusters must be non-null"; + clusterPrivileges = remoteClusterGroup.get(PRIVILEGES.getPreferredName()).toArray(new String[0]); + remoteClusterAliases = remoteClusterGroup.get(CLUSTERS.getPreferredName()).toArray(new String[0]); + remoteClusterAliasMatcher = StringMatcher.of(remoteClusterAliases); + } + /** * @param clusterPrivileges The list of cluster privileges that are allowed for the remote cluster. must not be null or empty. * @param remoteClusterAliases The list of remote clusters that the privileges apply to. must not be null or empty. @@ -53,10 +65,14 @@ public RemoteClusterPermissionGroup(String[] clusterPrivileges, String[] remoteC throw new IllegalArgumentException("remote cluster groups must not be null or empty"); } if (Arrays.stream(clusterPrivileges).anyMatch(s -> Strings.hasText(s) == false)) { - throw new IllegalArgumentException("remote_cluster privileges must contain valid non-empty, non-null values"); + throw new IllegalArgumentException( + "remote_cluster privileges must contain valid non-empty, non-null values " + Arrays.toString(clusterPrivileges) + ); } if (Arrays.stream(remoteClusterAliases).anyMatch(s -> Strings.hasText(s) == false)) { - throw new IllegalArgumentException("remote_cluster clusters aliases must contain valid non-empty, non-null values"); + throw new IllegalArgumentException( + "remote_cluster clusters aliases must contain valid non-empty, non-null values " + Arrays.toString(remoteClusterAliases) + ); } this.clusterPrivileges = clusterPrivileges; @@ -86,11 +102,24 @@ public String[] remoteClusterAliases() { return Arrays.copyOf(remoteClusterAliases, remoteClusterAliases.length); } + /** + * Converts the group to a map representation. + * @return A map representation of the group. + */ + public Map> toMap() { + return Map.of( + PRIVILEGES.getPreferredName(), + Arrays.asList(clusterPrivileges), + CLUSTERS.getPreferredName(), + Arrays.asList(remoteClusterAliases) + ); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.array(RoleDescriptor.Fields.PRIVILEGES.getPreferredName(), clusterPrivileges); - builder.array(RoleDescriptor.Fields.CLUSTERS.getPreferredName(), remoteClusterAliases); + builder.array(PRIVILEGES.getPreferredName(), clusterPrivileges); + builder.array(CLUSTERS.getPreferredName(), remoteClusterAliases); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissions.java index 0d8880c33720b..1928cf117dde3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissions.java @@ -29,13 +29,19 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; +import static org.elasticsearch.TransportVersions.ROLE_MONITOR_STATS; + /** * Represents the set of permissions for remote clusters. This is intended to be the model for both the {@link RoleDescriptor} - * and {@link Role}. This model is not intended to be sent to a remote cluster, but can be (wire) serialized within a single cluster - * as well as the Xcontent serialization for the REST API and persistence of the role in the security index. The privileges modeled here - * will be converted to the appropriate cluster privileges when sent to a remote cluster. + * and {@link Role}. This model is intended to be converted to local cluster permissions + * {@link #collapseAndRemoveUnsupportedPrivileges(String, TransportVersion)} before sent to the remote cluster. This model also be included + * in the role descriptors for (normal) API keys sent between nodes/clusters. In both cases the outbound transport version can be used to + * remove permissions that are not available to older nodes or clusters. The methods {@link #removeUnsupportedPrivileges(TransportVersion)} + * and {@link #collapseAndRemoveUnsupportedPrivileges(String, TransportVersion)} are used to aid in ensuring correct privileges per + * transport version. * For example, on the local/querying cluster this model represents the following: * * "remote_cluster" : [ @@ -49,15 +55,18 @@ * } * ] * - * when sent to the remote cluster "clusterA", the privileges will be converted to the appropriate cluster privileges. For example: + * (RCS 2.0) when sent to the remote cluster "clusterA", the privileges will be converted to the appropriate cluster privileges. + * For example: * * "cluster": ["foo"] * - * and when sent to the remote cluster "clusterB", the privileges will be converted to the appropriate cluster privileges. For example: + * and (RCS 2.0) when sent to the remote cluster "clusterB", the privileges will be converted to the appropriate cluster privileges. + * For example: * * "cluster": ["bar"] * - * If the remote cluster does not support the privilege, as determined by the remote cluster version, the privilege will be not be sent. + * For normal API keys and their role descriptors :If the remote cluster does not support the privilege, the privilege will be not be sent. + * Upstream code performs the removal, but this class owns the business logic for how to remove per outbound version. */ public class RemoteClusterPermissions implements NamedWriteable, ToXContentObject { @@ -70,19 +79,33 @@ public class RemoteClusterPermissions implements NamedWriteable, ToXContentObjec // package private non-final for testing static Map> allowedRemoteClusterPermissions = Map.of( ROLE_REMOTE_CLUSTER_PRIVS, - Set.of(ClusterPrivilegeResolver.MONITOR_ENRICH.name()) + Set.of(ClusterPrivilegeResolver.MONITOR_ENRICH.name()), + ROLE_MONITOR_STATS, + Set.of(ClusterPrivilegeResolver.MONITOR_STATS.name()) ); + static final TransportVersion lastTransportVersionPermission = allowedRemoteClusterPermissions.keySet() + .stream() + .max(TransportVersion::compareTo) + .orElseThrow(); public static final RemoteClusterPermissions NONE = new RemoteClusterPermissions(); public static Set getSupportedRemoteClusterPermissions() { - return allowedRemoteClusterPermissions.values().stream().flatMap(Set::stream).collect(Collectors.toSet()); + return allowedRemoteClusterPermissions.values().stream().flatMap(Set::stream).collect(Collectors.toCollection(TreeSet::new)); } public RemoteClusterPermissions(StreamInput in) throws IOException { remoteClusterPermissionGroups = in.readNamedWriteableCollectionAsList(RemoteClusterPermissionGroup.class); } + public RemoteClusterPermissions(List>> remoteClusters) { + remoteClusterPermissionGroups = new ArrayList<>(); + for (Map> remoteCluster : remoteClusters) { + RemoteClusterPermissionGroup remoteClusterPermissionGroup = new RemoteClusterPermissionGroup(remoteCluster); + remoteClusterPermissionGroups.add(remoteClusterPermissionGroup); + } + } + public RemoteClusterPermissions() { remoteClusterPermissionGroups = new ArrayList<>(); } @@ -97,10 +120,64 @@ public RemoteClusterPermissions addGroup(RemoteClusterPermissionGroup remoteClus } /** - * Gets the privilege names for the remote cluster. This method will collapse all groups to single String[] all lowercase - * and will only return the appropriate privileges for the provided remote cluster version. + * Will remove any unsupported privileges for the provided outbound version. This method will not modify the current instance. + * This is useful for (normal) API keys role descriptors to help ensure that we don't send unsupported privileges. The result of + * this method may result in no groups if all privileges are removed. {@link #hasAnyPrivileges()} can be used to check if there are + * any privileges left. + * @param outboundVersion The version by which to remove unsupported privileges, this is typically the version of the remote cluster + * @return a new instance of RemoteClusterPermissions with the unsupported privileges removed */ - public String[] privilegeNames(final String remoteClusterAlias, TransportVersion remoteClusterVersion) { + public RemoteClusterPermissions removeUnsupportedPrivileges(TransportVersion outboundVersion) { + Objects.requireNonNull(outboundVersion, "outboundVersion must not be null"); + if (outboundVersion.onOrAfter(lastTransportVersionPermission)) { + return this; + } + RemoteClusterPermissions copyForOutboundVersion = new RemoteClusterPermissions(); + Set allowedPermissionsPerVersion = getAllowedPermissionsPerVersion(outboundVersion); + for (RemoteClusterPermissionGroup group : remoteClusterPermissionGroups) { + String[] privileges = group.clusterPrivileges(); + List outboundPrivileges = new ArrayList<>(privileges.length); + for (String privilege : privileges) { + if (allowedPermissionsPerVersion.contains(privilege.toLowerCase(Locale.ROOT))) { + outboundPrivileges.add(privilege); + } + } + if (outboundPrivileges.isEmpty() == false) { + RemoteClusterPermissionGroup outboundGroup = new RemoteClusterPermissionGroup( + outboundPrivileges.toArray(new String[0]), + group.remoteClusterAliases() + ); + copyForOutboundVersion.addGroup(outboundGroup); + if (logger.isDebugEnabled()) { + if (group.equals(outboundGroup) == false) { + logger.debug( + "Removed unsupported remote cluster permissions. Remaining {} for remote cluster [{}] for version [{}]." + + "Due to the remote cluster version, only the following permissions are allowed: {}", + outboundPrivileges, + group.remoteClusterAliases(), + outboundVersion, + allowedPermissionsPerVersion + ); + } + } + } else { + logger.debug( + "Removed all remote cluster permissions for remote cluster [{}]. " + + "Due to the remote cluster version, only the following permissions are allowed: {}", + group.remoteClusterAliases(), + allowedPermissionsPerVersion + ); + } + } + return copyForOutboundVersion; + } + + /** + * Gets all the privilege names for the remote cluster. This method will collapse all groups to single String[] all lowercase + * and will only return the appropriate privileges for the provided remote cluster version. This is useful for RCS 2.0 to ensure + * that we properly convert all the remote_cluster -> cluster privileges per remote cluster. + */ + public String[] collapseAndRemoveUnsupportedPrivileges(final String remoteClusterAlias, TransportVersion outboundVersion) { // get all privileges for the remote cluster Set groupPrivileges = remoteClusterPermissionGroups.stream() @@ -111,13 +188,7 @@ public String[] privilegeNames(final String remoteClusterAlias, TransportVersion .collect(Collectors.toSet()); // find all the privileges that are allowed for the remote cluster version - Set allowedPermissionsPerVersion = allowedRemoteClusterPermissions.entrySet() - .stream() - .filter((entry) -> entry.getKey().onOrBefore(remoteClusterVersion)) - .map(Map.Entry::getValue) - .flatMap(Set::stream) - .map(s -> s.toLowerCase(Locale.ROOT)) - .collect(Collectors.toSet()); + Set allowedPermissionsPerVersion = getAllowedPermissionsPerVersion(outboundVersion); // intersect the two sets to get the allowed privileges for the remote cluster version Set allowedPrivileges = new HashSet<>(groupPrivileges); @@ -137,13 +208,21 @@ public String[] privilegeNames(final String remoteClusterAlias, TransportVersion return allowedPrivileges.stream().sorted().toArray(String[]::new); } + /** + * Converts this object to it's {@link Map} representation. + * @return a list of maps representing the remote cluster permissions + */ + public List>> toMap() { + return remoteClusterPermissionGroups.stream().map(RemoteClusterPermissionGroup::toMap).toList(); + } + /** * Validates the remote cluster permissions (regardless of remote cluster version). * This method will throw an {@link IllegalArgumentException} if the permissions are invalid. * Generally, this method is just a safety check and validity should be checked before adding the permissions to this class. */ public void validate() { - assert hasPrivileges(); + assert hasAnyPrivileges(); Set invalid = getUnsupportedPrivileges(); if (invalid.isEmpty() == false) { throw new IllegalArgumentException( @@ -173,11 +252,11 @@ private Set getUnsupportedPrivileges() { return invalid; } - public boolean hasPrivileges(final String remoteClusterAlias) { + public boolean hasAnyPrivileges(final String remoteClusterAlias) { return remoteClusterPermissionGroups.stream().anyMatch(remoteIndicesGroup -> remoteIndicesGroup.hasPrivileges(remoteClusterAlias)); } - public boolean hasPrivileges() { + public boolean hasAnyPrivileges() { return remoteClusterPermissionGroups.isEmpty() == false; } @@ -185,6 +264,16 @@ public List groups() { return Collections.unmodifiableList(remoteClusterPermissionGroups); } + private Set getAllowedPermissionsPerVersion(TransportVersion outboundVersion) { + return allowedRemoteClusterPermissions.entrySet() + .stream() + .filter((entry) -> entry.getKey().onOrBefore(outboundVersion)) + .map(Map.Entry::getValue) + .flatMap(Set::stream) + .map(s -> s.toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { for (RemoteClusterPermissionGroup remoteClusterPermissionGroup : remoteClusterPermissionGroups) { @@ -220,4 +309,5 @@ public String toString() { public String getWriteableName() { return NAME; } + } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java index d8d56a4fbb247..f52f8f85f006d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java @@ -283,7 +283,7 @@ public Builder addRemoteIndicesGroup( public Builder addRemoteClusterPermissions(RemoteClusterPermissions remoteClusterPermissions) { Objects.requireNonNull(remoteClusterPermissions, "remoteClusterPermissions must not be null"); assert this.remoteClusterPermissions == null : "addRemoteClusterPermissions should only be called once"; - if (remoteClusterPermissions.hasPrivileges()) { + if (remoteClusterPermissions.hasAnyPrivileges()) { remoteClusterPermissions.validate(); } this.remoteClusterPermissions = remoteClusterPermissions; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java index 08c86c5f71f4f..0ec9d2a48316a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java @@ -210,7 +210,7 @@ public RoleDescriptorsIntersection getRoleDescriptorsIntersectionForRemoteCluste final RemoteIndicesPermission remoteIndicesPermission = this.remoteIndicesPermission.forCluster(remoteClusterAlias); if (remoteIndicesPermission.remoteIndicesGroups().isEmpty() - && remoteClusterPermissions.hasPrivileges(remoteClusterAlias) == false) { + && remoteClusterPermissions.hasAnyPrivileges(remoteClusterAlias) == false) { return RoleDescriptorsIntersection.EMPTY; } @@ -224,7 +224,7 @@ public RoleDescriptorsIntersection getRoleDescriptorsIntersectionForRemoteCluste return new RoleDescriptorsIntersection( new RoleDescriptor( REMOTE_USER_ROLE_NAME, - remoteClusterPermissions.privilegeNames(remoteClusterAlias, remoteClusterVersion), + remoteClusterPermissions.collapseAndRemoveUnsupportedPrivileges(remoteClusterAlias, remoteClusterVersion), // The role descriptors constructed here may be cached in raw byte form, using a hash of their content as a // cache key; we therefore need deterministic order when constructing them here, to ensure cache hits for // equivalent role descriptors diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java index 3d1b378f4f51e..00d45fb135fb2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java @@ -110,6 +110,8 @@ public class ClusterPrivilegeResolver { private static final Set MONITOR_WATCHER_PATTERN = Set.of("cluster:monitor/xpack/watcher/*"); private static final Set MONITOR_ROLLUP_PATTERN = Set.of("cluster:monitor/xpack/rollup/*"); private static final Set MONITOR_ENRICH_PATTERN = Set.of("cluster:monitor/xpack/enrich/*", "cluster:admin/xpack/enrich/get"); + // intentionally cluster:monitor/stats* to match cluster:monitor/stats, cluster:monitor/stats[n] and cluster:monitor/stats/remote + private static final Set MONITOR_STATS_PATTERN = Set.of("cluster:monitor/stats*"); private static final Set ALL_CLUSTER_PATTERN = Set.of( "cluster:*", @@ -208,7 +210,11 @@ public class ClusterPrivilegeResolver { // esql enrich "cluster:monitor/xpack/enrich/esql/resolve_policy", "cluster:internal:data/read/esql/open_exchange", - "cluster:internal:data/read/esql/exchange" + "cluster:internal:data/read/esql/exchange", + // cluster stats for remote clusters + "cluster:monitor/stats/remote", + "cluster:monitor/stats", + "cluster:monitor/stats[n]" ); private static final Set CROSS_CLUSTER_REPLICATION_PATTERN = Set.of( RemoteClusterService.REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME, @@ -243,6 +249,7 @@ public class ClusterPrivilegeResolver { public static final NamedClusterPrivilege MONITOR_WATCHER = new ActionClusterPrivilege("monitor_watcher", MONITOR_WATCHER_PATTERN); public static final NamedClusterPrivilege MONITOR_ROLLUP = new ActionClusterPrivilege("monitor_rollup", MONITOR_ROLLUP_PATTERN); public static final NamedClusterPrivilege MONITOR_ENRICH = new ActionClusterPrivilege("monitor_enrich", MONITOR_ENRICH_PATTERN); + public static final NamedClusterPrivilege MONITOR_STATS = new ActionClusterPrivilege("monitor_stats", MONITOR_STATS_PATTERN); public static final NamedClusterPrivilege MANAGE = new ActionClusterPrivilege("manage", ALL_CLUSTER_PATTERN, ALL_SECURITY_PATTERN); public static final NamedClusterPrivilege MANAGE_INFERENCE = new ActionClusterPrivilege("manage_inference", MANAGE_INFERENCE_PATTERN); public static final NamedClusterPrivilege MANAGE_ML = new ActionClusterPrivilege("manage_ml", MANAGE_ML_PATTERN); @@ -424,6 +431,7 @@ public class ClusterPrivilegeResolver { MONITOR_WATCHER, MONITOR_ROLLUP, MONITOR_ENRICH, + MONITOR_STATS, MANAGE, MANAGE_CONNECTOR, MANAGE_INFERENCE, @@ -499,7 +507,7 @@ public static NamedClusterPrivilege resolve(String name) { + Strings.collectionToCommaDelimitedString(VALUES.keySet()) + "] or a pattern over one of the available " + "cluster actions"; - logger.debug(errorMessage); + logger.warn(errorMessage); throw new IllegalArgumentException(errorMessage); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 0028508e87f32..9fb43d3b450b8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -20,6 +20,9 @@ import org.elasticsearch.xpack.core.security.action.profile.SuggestProfilesAction; import org.elasticsearch.xpack.core.security.action.user.ProfileHasPrivilegesAction; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissionGroup; +import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions; +import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver; import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges; import org.elasticsearch.xpack.core.security.support.MetadataUtils; @@ -492,7 +495,15 @@ static RoleDescriptor kibanaSystem(String name) { getRemoteIndicesReadPrivileges("metrics-apm.*"), getRemoteIndicesReadPrivileges("traces-apm.*"), getRemoteIndicesReadPrivileges("traces-apm-*") }, - null, + new RemoteClusterPermissions().addGroup( + new RemoteClusterPermissionGroup( + RemoteClusterPermissions.getSupportedRemoteClusterPermissions() + .stream() + .filter(s -> s.equals(ClusterPrivilegeResolver.MONITOR_STATS.name())) + .toArray(String[]::new), + new String[] { "*" } + ) + ), null, "Grants access necessary for the Kibana system user to read from and write to the Kibana indices, " + "manage index templates and tokens, and check the availability of the Elasticsearch cluster. " diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CrossClusterApiKeyRoleDescriptorBuilderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CrossClusterApiKeyRoleDescriptorBuilderTests.java index 22590e155e642..1dfd68ea95485 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CrossClusterApiKeyRoleDescriptorBuilderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CrossClusterApiKeyRoleDescriptorBuilderTests.java @@ -10,11 +10,16 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.core.Strings; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.permission.ClusterPermission; import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions; +import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; +import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver; import java.io.IOException; import java.util.List; @@ -27,6 +32,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; public class CrossClusterApiKeyRoleDescriptorBuilderTests extends ESTestCase { @@ -356,9 +362,42 @@ public void testEmptyAccessIsNotAllowed() throws IOException { } public void testAPIKeyAllowsAllRemoteClusterPrivilegesForCCS() { - // if users can add remote cluster permissions to a role, then the APIKey should also allow that for that permission - // the inverse however, is not guaranteed. cross_cluster_search exists largely for internal use and is not exposed to the users role - assertTrue(Set.of(CCS_CLUSTER_PRIVILEGE_NAMES).containsAll(RemoteClusterPermissions.getSupportedRemoteClusterPermissions())); + // test to help ensure that at least 1 action that is allowed by the remote cluster permissions are supported by CCS + List actionsToTest = List.of("cluster:monitor/xpack/enrich/esql/resolve_policy", "cluster:monitor/stats/remote"); + // if you add new remote cluster permissions, please define an action we can test to help ensure it is supported by RCS 2.0 + assertThat(actionsToTest.size(), equalTo(RemoteClusterPermissions.getSupportedRemoteClusterPermissions().size())); + + for (String privilege : RemoteClusterPermissions.getSupportedRemoteClusterPermissions()) { + boolean actionPassesRemoteClusterPermissionCheck = false; + ClusterPrivilege clusterPrivilege = ClusterPrivilegeResolver.resolve(privilege); + // each remote cluster privilege has an action to test + for (String action : actionsToTest) { + if (clusterPrivilege.buildPermission(ClusterPermission.builder()) + .build() + .check(action, mock(TransportRequest.class), AuthenticationTestHelper.builder().build())) { + actionPassesRemoteClusterPermissionCheck = true; + break; + } + } + assertTrue( + "privilege [" + privilege + "] does not cover any actions among [" + actionsToTest + "]", + actionPassesRemoteClusterPermissionCheck + ); + } + // test that the actions pass the privilege check for CCS + for (String privilege : Set.of(CCS_CLUSTER_PRIVILEGE_NAMES)) { + boolean actionPassesRemoteCCSCheck = false; + ClusterPrivilege clusterPrivilege = ClusterPrivilegeResolver.resolve(privilege); + for (String action : actionsToTest) { + if (clusterPrivilege.buildPermission(ClusterPermission.builder()) + .build() + .check(action, mock(TransportRequest.class), AuthenticationTestHelper.builder().build())) { + actionPassesRemoteCCSCheck = true; + break; + } + } + assertTrue(actionPassesRemoteCCSCheck); + } } private static void assertRoleDescriptor( diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java index 97255502bc7be..239d48ca9c2e1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java @@ -104,7 +104,7 @@ public void testValidationErrorWithUnknownRemoteClusterPrivilegeName() { } request.putRemoteCluster(remoteClusterPermissions); assertValidationError("Invalid remote_cluster permissions found. Please remove the following: [", request); - assertValidationError("Only [monitor_enrich] are allowed", request); + assertValidationError("Only [monitor_enrich, monitor_stats] are allowed", request); } public void testValidationErrorWithEmptyClustersInRemoteIndices() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java index 66e246d1c8a50..c999c970a76da 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.transport.RemoteClusterPortSettings; +import org.elasticsearch.xcontent.ObjectPath; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -32,6 +33,7 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.User; +import org.hamcrest.Matchers; import java.io.IOException; import java.util.Arrays; @@ -42,6 +44,8 @@ import java.util.stream.Collectors; import static java.util.Map.entry; +import static org.elasticsearch.TransportVersions.ROLE_MONITOR_STATS; +import static org.elasticsearch.xpack.core.security.authc.Authentication.VERSION_API_KEY_ROLES_AS_BYTES; import static org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper.randomCrossClusterAccessSubjectInfo; import static org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfoTests.randomRoleDescriptorsIntersection; import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS; @@ -1070,7 +1074,7 @@ public void testMaybeRewriteMetadataForApiKeyRoleDescriptorsWithRemoteIndices() // pick a version before that of the authentication instance to force a rewrite final TransportVersion olderVersion = TransportVersionUtils.randomVersionBetween( random(), - Authentication.VERSION_API_KEY_ROLES_AS_BYTES, + VERSION_API_KEY_ROLES_AS_BYTES, TransportVersionUtils.getPreviousVersion(original.getEffectiveSubject().getTransportVersion()) ); @@ -1115,7 +1119,7 @@ public void testMaybeRewriteMetadataForApiKeyRoleDescriptorsWithRemoteCluster() // pick a version before that of the authentication instance to force a rewrite final TransportVersion olderVersion = TransportVersionUtils.randomVersionBetween( random(), - Authentication.VERSION_API_KEY_ROLES_AS_BYTES, + VERSION_API_KEY_ROLES_AS_BYTES, TransportVersionUtils.getPreviousVersion(original.getEffectiveSubject().getTransportVersion()) ); @@ -1135,6 +1139,84 @@ public void testMaybeRewriteMetadataForApiKeyRoleDescriptorsWithRemoteCluster() ); } + public void testMaybeRewriteMetadataForApiKeyRoleDescriptorsWithRemoteClusterRemovePrivs() throws IOException { + final String apiKeyId = randomAlphaOfLengthBetween(1, 10); + final String apiKeyName = randomAlphaOfLengthBetween(1, 10); + Map metadata = Map.ofEntries( + entry(AuthenticationField.API_KEY_ID_KEY, apiKeyId), + entry(AuthenticationField.API_KEY_NAME_KEY, apiKeyName), + entry(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY, new BytesArray(""" + {"base_role":{"cluster":["all"], + "remote_cluster":[{"privileges":["monitor_enrich", "monitor_stats"],"clusters":["*"]}] + }}""")), + entry(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, new BytesArray(""" + {"limited_by_role":{"cluster":["*"], + "remote_cluster":[{"privileges":["monitor_enrich", "monitor_stats"],"clusters":["*"]}] + }}""")) + ); + + final Authentication with2privs = AuthenticationTestHelper.builder() + .apiKey() + .metadata(metadata) + .transportVersion(TransportVersion.current()) + .build(); + + // pick a version that will only remove one of the two privileges + final TransportVersion olderVersion = TransportVersionUtils.randomVersionBetween( + random(), + ROLE_REMOTE_CLUSTER_PRIVS, + TransportVersionUtils.getPreviousVersion(ROLE_MONITOR_STATS) + ); + + Map rewrittenMetadata = with2privs.maybeRewriteForOlderVersion(olderVersion).getEffectiveSubject().getMetadata(); + assertThat(rewrittenMetadata.keySet(), equalTo(with2privs.getAuthenticatingSubject().getMetadata().keySet())); + + // only one of the two privileges are left after the rewrite + BytesReference baseRoleBytes = (BytesReference) rewrittenMetadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY); + Map baseRoleAsMap = XContentHelper.convertToMap(baseRoleBytes, false, XContentType.JSON).v2(); + assertThat(ObjectPath.eval("base_role.remote_cluster.0.privileges", baseRoleAsMap), Matchers.contains("monitor_enrich")); + assertThat(ObjectPath.eval("base_role.remote_cluster.0.clusters", baseRoleAsMap), notNullValue()); + BytesReference limitedByRoleBytes = (BytesReference) rewrittenMetadata.get( + AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY + ); + Map limitedByRoleAsMap = XContentHelper.convertToMap(limitedByRoleBytes, false, XContentType.JSON).v2(); + assertThat(ObjectPath.eval("limited_by_role.remote_cluster.0.privileges", limitedByRoleAsMap), Matchers.contains("monitor_enrich")); + assertThat(ObjectPath.eval("limited_by_role.remote_cluster.0.clusters", limitedByRoleAsMap), notNullValue()); + + // same version, but it removes the only defined privilege + metadata = Map.ofEntries( + entry(AuthenticationField.API_KEY_ID_KEY, apiKeyId), + entry(AuthenticationField.API_KEY_NAME_KEY, apiKeyName), + entry(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY, new BytesArray(""" + {"base_role":{"cluster":["all"], + "remote_cluster":[{"privileges":["monitor_stats"],"clusters":["*"]}] + }}""")), + entry(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, new BytesArray(""" + {"limited_by_role":{"cluster":["*"], + "remote_cluster":[{"privileges":["monitor_stats"],"clusters":["*"]}] + }}""")) + ); + + final Authentication with1priv = AuthenticationTestHelper.builder() + .apiKey() + .metadata(metadata) + .transportVersion(TransportVersion.current()) + .build(); + + rewrittenMetadata = with1priv.maybeRewriteForOlderVersion(olderVersion).getEffectiveSubject().getMetadata(); + assertThat(rewrittenMetadata.keySet(), equalTo(with1priv.getAuthenticatingSubject().getMetadata().keySet())); + + // the one privileges is removed after the rewrite, which removes the full "remote_cluster" object + baseRoleBytes = (BytesReference) rewrittenMetadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY); + baseRoleAsMap = XContentHelper.convertToMap(baseRoleBytes, false, XContentType.JSON).v2(); + assertThat(ObjectPath.eval("base_role.remote_cluster", baseRoleAsMap), nullValue()); + assertThat(ObjectPath.eval("base_role.cluster", baseRoleAsMap), notNullValue()); + limitedByRoleBytes = (BytesReference) rewrittenMetadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY); + limitedByRoleAsMap = XContentHelper.convertToMap(limitedByRoleBytes, false, XContentType.JSON).v2(); + assertThat(ObjectPath.eval("limited_by_role.remote_cluster", limitedByRoleAsMap), nullValue()); + assertThat(ObjectPath.eval("limited_by_role.cluster", limitedByRoleAsMap), notNullValue()); + } + public void testMaybeRemoveRemoteIndicesFromRoleDescriptors() { final boolean includeClusterPrivileges = randomBoolean(); final BytesReference roleWithoutRemoteIndices = new BytesArray(Strings.format(""" diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java index 7d8e851e1d68e..70b69ce9cbc04 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java @@ -542,6 +542,34 @@ public void testParseInvalidRemoteCluster() throws IOException { () -> RoleDescriptor.parserBuilder().build().parse("test", new BytesArray(q4), XContentType.JSON) ); assertThat(illegalArgumentException.getMessage(), containsString("remote cluster groups must not be null or empty")); + + // one invalid privilege + String q5 = """ + { + "remote_cluster": [ + { + "privileges": [ + "monitor_stats", "read_pipeline" + ], + "clusters": [ + "*" + ] + } + ] + }"""; + + ElasticsearchParseException parseException = expectThrows( + ElasticsearchParseException.class, + () -> RoleDescriptor.parserBuilder().build().parse("test", new BytesArray(q5), XContentType.JSON) + ); + assertThat( + parseException.getMessage(), + containsString( + "failed to parse remote_cluster for role [test]. " + + "[monitor_enrich, monitor_stats] are the only values allowed for [privileges] within [remote_cluster]. " + + "Found [monitor_stats, read_pipeline]" + ) + ); } public void testParsingFieldPermissionsUsesCache() throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionGroupTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionGroupTests.java index cd269bd1a97b3..0b99db826d540 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionGroupTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionGroupTests.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Locale; +import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -90,7 +91,7 @@ public void testInvalidValues() { ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, invalidClusterAlias); - assertEquals("remote_cluster clusters aliases must contain valid non-empty, non-null values", e.getMessage()); + assertThat(e.getMessage(), containsString("remote_cluster clusters aliases must contain valid non-empty, non-null values")); final ThrowingRunnable invalidPermission = randomFrom( () -> new RemoteClusterPermissionGroup(new String[] { null }, new String[] { "bar" }), @@ -100,7 +101,17 @@ public void testInvalidValues() { ); IllegalArgumentException e2 = expectThrows(IllegalArgumentException.class, invalidPermission); - assertEquals("remote_cluster privileges must contain valid non-empty, non-null values", e2.getMessage()); + assertThat(e2.getMessage(), containsString("remote_cluster privileges must contain valid non-empty, non-null values")); + } + + public void testToMap() { + String[] privileges = generateRandomStringArray(5, 5, false, false); + String[] clusters = generateRandomStringArray(5, 5, false, false); + RemoteClusterPermissionGroup remoteClusterPermissionGroup = new RemoteClusterPermissionGroup(privileges, clusters); + assertEquals( + Map.of("privileges", Arrays.asList(privileges), "clusters", Arrays.asList(clusters)), + remoteClusterPermissionGroup.toMap() + ); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionsTests.java index 5b5a895f12ae8..2c31965009273 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteClusterPermissionsTests.java @@ -15,6 +15,8 @@ import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.xcontent.XContentUtils; import org.junit.Before; import java.io.IOException; @@ -27,8 +29,11 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import static org.elasticsearch.TransportVersions.ROLE_MONITOR_STATS; import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.ROLE_REMOTE_CLUSTER_PRIVS; +import static org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions.lastTransportVersionPermission; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -85,13 +90,13 @@ public void testMatcher() { for (int i = 0; i < generateRandomGroups(true).size(); i++) { String[] clusters = groupClusters.get(i); for (String cluster : clusters) { - assertTrue(remoteClusterPermission.hasPrivileges(cluster)); - assertFalse(remoteClusterPermission.hasPrivileges(randomAlphaOfLength(20))); + assertTrue(remoteClusterPermission.hasAnyPrivileges(cluster)); + assertFalse(remoteClusterPermission.hasAnyPrivileges(randomAlphaOfLength(20))); } } } - public void testPrivilegeNames() { + public void testCollapseAndRemoveUnsupportedPrivileges() { Map> original = RemoteClusterPermissions.allowedRemoteClusterPermissions; try { // create random groups with random privileges for random clusters @@ -108,7 +113,7 @@ public void testPrivilegeNames() { String[] privileges = groupPrivileges.get(i); String[] clusters = groupClusters.get(i); for (String cluster : clusters) { - String[] found = remoteClusterPermission.privilegeNames(cluster, TransportVersion.current()); + String[] found = remoteClusterPermission.collapseAndRemoveUnsupportedPrivileges(cluster, TransportVersion.current()); Arrays.sort(found); // ensure all lowercase since the privilege names are case insensitive and the method will result in lowercase for (int j = 0; j < privileges.length; j++) { @@ -126,13 +131,14 @@ public void testPrivilegeNames() { // create random groups with random privileges for random clusters List randomGroups = generateRandomGroups(true); // replace a random value with one that is allowed - groupPrivileges.get(0)[0] = "monitor_enrich"; + String singleValidPrivilege = randomFrom(RemoteClusterPermissions.allowedRemoteClusterPermissions.get(TransportVersion.current())); + groupPrivileges.get(0)[0] = singleValidPrivilege; for (int i = 0; i < randomGroups.size(); i++) { String[] privileges = groupPrivileges.get(i); String[] clusters = groupClusters.get(i); for (String cluster : clusters) { - String[] found = remoteClusterPermission.privilegeNames(cluster, TransportVersion.current()); + String[] found = remoteClusterPermission.collapseAndRemoveUnsupportedPrivileges(cluster, TransportVersion.current()); Arrays.sort(found); // ensure all lowercase since the privilege names are case insensitive and the method will result in lowercase for (int j = 0; j < privileges.length; j++) { @@ -149,7 +155,7 @@ public void testPrivilegeNames() { assertFalse(Arrays.equals(privileges, found)); if (i == 0) { // ensure that for the current version we only find the valid "monitor_enrich" - assertThat(Set.of(found), equalTo(Set.of("monitor_enrich"))); + assertThat(Set.of(found), equalTo(Set.of(singleValidPrivilege))); } else { // all other groups should be found to not have any privileges assertTrue(found.length == 0); @@ -159,21 +165,26 @@ public void testPrivilegeNames() { } } - public void testMonitorEnrichPerVersion() { - // test monitor_enrich before, after and on monitor enrich version - String[] privileges = randomBoolean() ? new String[] { "monitor_enrich" } : new String[] { "monitor_enrich", "foo", "bar" }; + public void testPermissionsPerVersion() { + testPermissionPerVersion("monitor_enrich", ROLE_REMOTE_CLUSTER_PRIVS); + testPermissionPerVersion("monitor_stats", ROLE_MONITOR_STATS); + } + + private void testPermissionPerVersion(String permission, TransportVersion version) { + // test permission before, after and on the version + String[] privileges = randomBoolean() ? new String[] { permission } : new String[] { permission, "foo", "bar" }; String[] before = new RemoteClusterPermissions().addGroup(new RemoteClusterPermissionGroup(privileges, new String[] { "*" })) - .privilegeNames("*", TransportVersionUtils.getPreviousVersion(ROLE_REMOTE_CLUSTER_PRIVS)); - // empty set since monitor_enrich is not allowed in the before version + .collapseAndRemoveUnsupportedPrivileges("*", TransportVersionUtils.getPreviousVersion(version)); + // empty set since permissions is not allowed in the before version assertThat(Set.of(before), equalTo(Collections.emptySet())); String[] on = new RemoteClusterPermissions().addGroup(new RemoteClusterPermissionGroup(privileges, new String[] { "*" })) - .privilegeNames("*", ROLE_REMOTE_CLUSTER_PRIVS); - // only monitor_enrich since the other values are not allowed - assertThat(Set.of(on), equalTo(Set.of("monitor_enrich"))); + .collapseAndRemoveUnsupportedPrivileges("*", version); + // the permission is found on that provided version + assertThat(Set.of(on), equalTo(Set.of(permission))); String[] after = new RemoteClusterPermissions().addGroup(new RemoteClusterPermissionGroup(privileges, new String[] { "*" })) - .privilegeNames("*", TransportVersion.current()); - // only monitor_enrich since the other values are not allowed - assertThat(Set.of(after), equalTo(Set.of("monitor_enrich"))); + .collapseAndRemoveUnsupportedPrivileges("*", TransportVersion.current()); + // current version (after the version) has the permission + assertThat(Set.of(after), equalTo(Set.of(permission))); } public void testValidate() { @@ -181,12 +192,70 @@ public void testValidate() { // random values not allowed IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> remoteClusterPermission.validate()); assertTrue(error.getMessage().contains("Invalid remote_cluster permissions found. Please remove the following:")); - assertTrue(error.getMessage().contains("Only [monitor_enrich] are allowed")); + assertTrue(error.getMessage().contains("Only [monitor_enrich, monitor_stats] are allowed")); new RemoteClusterPermissions().addGroup(new RemoteClusterPermissionGroup(new String[] { "monitor_enrich" }, new String[] { "*" })) .validate(); // no error } + public void testToMap() { + RemoteClusterPermissions remoteClusterPermissions = new RemoteClusterPermissions(); + List groups = generateRandomGroups(randomBoolean()); + for (int i = 0; i < groups.size(); i++) { + remoteClusterPermissions.addGroup(groups.get(i)); + } + List>> asAsMap = remoteClusterPermissions.toMap(); + RemoteClusterPermissions remoteClusterPermissionsAsMap = new RemoteClusterPermissions(asAsMap); + assertEquals(remoteClusterPermissions, remoteClusterPermissionsAsMap); + } + + public void testRemoveUnsupportedPrivileges() { + RemoteClusterPermissions remoteClusterPermissions = new RemoteClusterPermissions(); + RemoteClusterPermissionGroup group = new RemoteClusterPermissionGroup(new String[] { "monitor_enrich" }, new String[] { "*" }); + remoteClusterPermissions.addGroup(group); + // this privilege is allowed by versions, so nothing should be removed + assertEquals(remoteClusterPermissions, remoteClusterPermissions.removeUnsupportedPrivileges(ROLE_REMOTE_CLUSTER_PRIVS)); + assertEquals(remoteClusterPermissions, remoteClusterPermissions.removeUnsupportedPrivileges(ROLE_MONITOR_STATS)); + + remoteClusterPermissions = new RemoteClusterPermissions(); + if (randomBoolean()) { + group = new RemoteClusterPermissionGroup(new String[] { "monitor_stats" }, new String[] { "*" }); + } else { + // if somehow duplicates end up here, they should not influence removal + group = new RemoteClusterPermissionGroup(new String[] { "monitor_stats", "monitor_stats" }, new String[] { "*" }); + } + remoteClusterPermissions.addGroup(group); + // this single newer privilege is not allowed in the older version, so it should result in an object with no groups + assertNotEquals(remoteClusterPermissions, remoteClusterPermissions.removeUnsupportedPrivileges(ROLE_REMOTE_CLUSTER_PRIVS)); + assertFalse(remoteClusterPermissions.removeUnsupportedPrivileges(ROLE_REMOTE_CLUSTER_PRIVS).hasAnyPrivileges()); + assertEquals(remoteClusterPermissions, remoteClusterPermissions.removeUnsupportedPrivileges(ROLE_MONITOR_STATS)); + + int groupCount = randomIntBetween(1, 5); + remoteClusterPermissions = new RemoteClusterPermissions(); + group = new RemoteClusterPermissionGroup(new String[] { "monitor_enrich", "monitor_stats" }, new String[] { "*" }); + for (int i = 0; i < groupCount; i++) { + remoteClusterPermissions.addGroup(group); + } + // one of the newer privilege is not allowed in the older version, so it should result in a group with only the allowed privilege + RemoteClusterPermissions expected = new RemoteClusterPermissions(); + for (int i = 0; i < groupCount; i++) { + expected.addGroup(new RemoteClusterPermissionGroup(new String[] { "monitor_enrich" }, new String[] { "*" })); + } + assertEquals(expected, remoteClusterPermissions.removeUnsupportedPrivileges(ROLE_REMOTE_CLUSTER_PRIVS)); + // both privileges allowed in the newer version, so it should not change the permission + assertEquals(remoteClusterPermissions, remoteClusterPermissions.removeUnsupportedPrivileges(ROLE_MONITOR_STATS)); + } + + public void testShortCircuitRemoveUnsupportedPrivileges() { + RemoteClusterPermissions remoteClusterPermissions = new RemoteClusterPermissions(); + assertSame(remoteClusterPermissions, remoteClusterPermissions.removeUnsupportedPrivileges(TransportVersion.current())); + assertSame(remoteClusterPermissions, remoteClusterPermissions.removeUnsupportedPrivileges(lastTransportVersionPermission)); + assertNotSame( + remoteClusterPermissions, + remoteClusterPermissions.removeUnsupportedPrivileges(TransportVersionUtils.getPreviousVersion(lastTransportVersionPermission)) + ); + } + private List generateRandomGroups(boolean fuzzyCluster) { clean(); List groups = new ArrayList<>(); @@ -216,22 +285,48 @@ protected Writeable.Reader instanceReader() { @Override protected RemoteClusterPermissions createTestInstance() { + Set all = RemoteClusterPermissions.allowedRemoteClusterPermissions.values() + .stream() + .flatMap(Set::stream) + .collect(Collectors.toSet()); + List randomPermission = randomList(1, all.size(), () -> randomFrom(all)); return new RemoteClusterPermissions().addGroup( - new RemoteClusterPermissionGroup(new String[] { "monitor_enrich" }, new String[] { "*" }) + new RemoteClusterPermissionGroup(randomPermission.toArray(new String[0]), new String[] { "*" }) ); } @Override protected RemoteClusterPermissions mutateInstance(RemoteClusterPermissions instance) throws IOException { return new RemoteClusterPermissions().addGroup( - new RemoteClusterPermissionGroup(new String[] { "monitor_enrich" }, new String[] { "*" }) + new RemoteClusterPermissionGroup(new String[] { "monitor_enrich", "monitor_stats" }, new String[] { "*" }) ).addGroup(new RemoteClusterPermissionGroup(new String[] { "foobar" }, new String[] { "*" })); } @Override protected RemoteClusterPermissions doParseInstance(XContentParser parser) throws IOException { - // fromXContent/parsing isn't supported since we still do old school manual parsing of the role descriptor - return createTestInstance(); + // fromXContent/object parsing isn't supported since we still do old school manual parsing of the role descriptor + // so this test is silly because it only tests we know how to manually parse the test instance in this test + // this is needed since we want the other parts from the AbstractXContentSerializingTestCase suite + RemoteClusterPermissions remoteClusterPermissions = new RemoteClusterPermissions(); + String[] privileges = null; + String[] clusters = null; + XContentParser.Token token; + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.START_OBJECT) { + continue; + } + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (RoleDescriptor.Fields.PRIVILEGES.match(currentFieldName, parser.getDeprecationHandler())) { + privileges = XContentUtils.readStringArray(parser, false); + + } else if (RoleDescriptor.Fields.CLUSTERS.match(currentFieldName, parser.getDeprecationHandler())) { + clusters = XContentUtils.readStringArray(parser, false); + } + } + remoteClusterPermissions.addGroup(new RemoteClusterPermissionGroup(privileges, clusters)); + return remoteClusterPermissions; } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 26b306d6f1334..58248121ddbad 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -2833,7 +2833,7 @@ public void testSuperuserRole() { is(false) ); assertThat( - superuserRole.remoteCluster().privilegeNames("*", TransportVersion.current()), + superuserRole.remoteCluster().collapseAndRemoveUnsupportedPrivileges("*", TransportVersion.current()), equalTo(RemoteClusterPermissions.getSupportedRemoteClusterPermissions().toArray(new String[0])) ); } diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle index fdaddbc1f9290..f96eff5f933c4 100644 --- a/x-pack/plugin/enrich/qa/rest/build.gradle +++ b/x-pack/plugin/enrich/qa/rest/build.gradle @@ -32,3 +32,4 @@ testClusters.configureEach { setting 'xpack.security.enabled', 'false' requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.4.0") } + diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java index 4076acdb7e7b8..d7ae438bc3189 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java @@ -174,18 +174,7 @@ public String getWriteableName() { @Override protected NodeInfo info() { - return NodeInfo.create( - this, - FieldAttribute::new, - parentName, - name(), - dataType(), - field, - (String) null, - nullable(), - id(), - synthetic() - ); + return NodeInfo.create(this, FieldAttribute::new, parentName, name(), field, nullable(), id(), synthetic()); } public String parentName() { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java index 3641812cd6cad..6e4e9292bfc99 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java @@ -147,26 +147,7 @@ protected String label() { @Override protected NodeInfo info() { - return NodeInfo.create( - this, - (source, name, dataType, qualifier, nullability, id, synthetic, searchable1) -> new MetadataAttribute( - source, - name, - dataType, - qualifier, - nullability, - id, - synthetic, - searchable1 - ), - name(), - dataType(), - (String) null, - nullable(), - id(), - synthetic(), - searchable - ); + return NodeInfo.create(this, MetadataAttribute::new, name(), dataType(), nullable(), id(), synthetic(), searchable); } public boolean searchable() { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java index 3626c5d26f235..404cd75edd5e4 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java @@ -110,24 +110,7 @@ protected Attribute clone(Source source, String name, DataType dataType, Nullabi @Override protected NodeInfo info() { - return NodeInfo.create( - this, - (source, name, dataType, qualifier, nullability, id, synthetic) -> new ReferenceAttribute( - source, - name, - dataType, - qualifier, - nullability, - id, - synthetic - ), - name(), - dataType(), - (String) null, - nullable(), - id(), - synthetic() - ); + return NodeInfo.create(this, ReferenceAttribute::new, name(), dataType(), nullable(), id(), synthetic()); } @Override diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index c5c3788fbce47..bc465e7e9b64c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -352,6 +352,16 @@ public static List> getValuesList(Iterator> values return valuesList; } + public static List> getValuesList(Iterable> values) { + var valuesList = new ArrayList>(); + values.iterator().forEachRemaining(row -> { + var rowValues = new ArrayList<>(); + row.iterator().forEachRemaining(rowValues::add); + valuesList.add(rowValues); + }); + return valuesList; + } + public static List withDefaultLimitWarning(List warnings) { List result = warnings == null ? new ArrayList<>() : new ArrayList<>(warnings); result.add("No limit defined, adding default limit of [1000]"); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec index b0578aa1a4ed0..c35f4c19cc347 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec @@ -6,11 +6,11 @@ matchWithField required_capability: match_function // tag::match-with-field[] -from books -| where match(author, "Faulkner") -| keep book_no, author -| sort book_no -| limit 5; +FROM books +| WHERE MATCH(author, "Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5; // end::match-with-field[] // tag::match-with-field-result[] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec index 18bb7cdf866c9..7b55ece964b89 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec @@ -6,11 +6,11 @@ matchWithField required_capability: match_operator_colon // tag::match-with-field[] -from books -| where author:"Faulkner" -| keep book_no, author -| sort book_no -| limit 5; +FROM books +| WHERE author:"Faulkner" +| KEEP book_no, author +| SORT book_no +| LIMIT 5; // end::match-with-field[] // tag::match-with-field-result[] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec index 6dc03d0debcfa..3e92e55928d64 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec @@ -6,11 +6,11 @@ qstrWithField required_capability: qstr_function // tag::qstr-with-field[] -from books -| where qstr("author: Faulkner") -| keep book_no, author -| sort book_no -| limit 5; +FROM books +| WHERE QSTR("author: Faulkner") +| KEEP book_no, author +| SORT book_no +| LIMIT 5; // end::qstr-with-field[] // tag::qstr-with-field-result[] diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java index 00efeb37a033b..7ae45497f7297 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java @@ -26,16 +26,21 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.xpack.core.esql.action.ColumnInfo; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.plugin.TransportEsqlQueryAction; import org.junit.After; +import java.util.ArrayList; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.concurrent.TimeUnit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @TestLogging(value = "org.elasticsearch.xpack.esql.session:DEBUG", reason = "to better understand planning") @@ -204,4 +209,28 @@ protected static QueryPragmas randomPragmas() { protected static boolean canUseQueryPragmas() { return Build.current().isSnapshot(); } + + protected static void assertColumnNames(List actualColumns, List expectedNames) { + assertThat(actualColumns.stream().map(ColumnInfo::name).toList(), equalTo(expectedNames)); + } + + protected static void assertColumnTypes(List actualColumns, List expectedTypes) { + assertThat(actualColumns.stream().map(ColumnInfo::outputType).toList(), equalTo(expectedTypes)); + } + + protected static void assertValues(Iterator> actualValues, Iterable> expectedValues) { + assertThat(getValuesList(actualValues), equalTo(getValuesList(expectedValues))); + } + + protected static void assertValuesInAnyOrder(Iterator> actualValues, Iterable> expectedValues) { + List> items = new ArrayList<>(); + for (Iterable outter : expectedValues) { + var item = new ArrayList<>(); + for (var inner : outter) { + item.add(inner); + } + items.add(item); + } + assertThat(getValuesList(actualValues), containsInAnyOrder(items.toArray())); + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java index fdb4ecd9b98d7..b86c46fd3fa7a 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.plugin; -import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; @@ -15,20 +14,15 @@ import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; -import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; -import org.elasticsearch.xpack.esql.core.type.DataType; import org.junit.Before; import java.util.List; -import static org.elasticsearch.test.ListMatcher.matchesList; -import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.Matchers.equalTo; @TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug") public class MatchOperatorIT extends AbstractEsqlIntegTestCase { @@ -40,7 +34,7 @@ public void setupIndex() { @Override protected EsqlQueryResponse run(EsqlQueryRequest request) { - assumeTrue("match operator available in snapshot builds only", Build.current().isSnapshot()); + assumeTrue("match operator capability not available", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled()); return super.run(request); } @@ -53,11 +47,9 @@ public void testSimpleWhereMatch() { """; try (var resp = run(query)) { - assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of("id"))); - assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of("INTEGER"))); - // values - List> values = getValuesList(resp); - assertMap(values, matchesList().item(List.of(1)).item(List.of(6))); + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(1), List.of(6))); } } @@ -70,11 +62,9 @@ public void testCombinedWhereMatch() { """; try (var resp = run(query)) { - assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of(("id")))); - assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of(("INTEGER")))); - // values - List> values = getValuesList(resp); - assertMap(values, matchesList().item(List.of(6))); + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(6))); } } @@ -87,12 +77,9 @@ public void testMultipleMatch() { """; try (var resp = run(query)) { - assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of(("id")))); - assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of(("INTEGER")))); - // values - List> values = getValuesList(resp); - assertThat(values.size(), equalTo(2)); - assertMap(values, matchesList().item(List.of(1)).item(List.of(6))); + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(1), List.of(6))); } } @@ -121,11 +108,9 @@ public void testNotWhereMatch() { """; try (var resp = run(query)) { - assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of(("id")))); - assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of(("INTEGER")))); - // values - List> values = getValuesList(resp); - assertMap(values, matchesList().item(List.of(5))); + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(5))); } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java index e7da83a40fb20..03af16d29e9b4 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java @@ -13,18 +13,12 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; -import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; -import org.elasticsearch.xpack.esql.core.type.DataType; import org.junit.Before; import java.util.List; -import static org.elasticsearch.test.ListMatcher.matchesList; -import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.Matchers.equalTo; public class QueryStringIT extends AbstractEsqlIntegTestCase { @@ -42,11 +36,9 @@ public void testSimpleQueryString() { """; try (var resp = run(query)) { - assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of("id"))); - assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of("INTEGER"))); - // values - List> values = getValuesList(resp); - assertMap(values, matchesList().item(List.of(1)).item(List.of(3)).item(List.of(4)).item(List.of(5))); + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValues(resp.values(), List.of(List.of(1), List.of(3), List.of(4), List.of(5))); } } @@ -58,11 +50,9 @@ public void testMultiFieldQueryString() { """; try (var resp = run(query)) { - assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of("id"))); - assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of("INTEGER"))); - // values - List> values = getValuesList(resp); - assertThat(values.size(), equalTo(5)); + assertColumnNames(resp.columns(), List.of("id")); + assertColumnTypes(resp.columns(), List.of("integer")); + assertValuesInAnyOrder(resp.values(), List.of(List.of(1), List.of(2), List.of(3), List.of(4), List.of(5))); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java index b8a64be5dfd35..82f0ebf316508 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java @@ -21,8 +21,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; -import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; -import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttributeTests; import org.elasticsearch.xpack.esql.core.expression.UnresolvedNamedExpression; @@ -114,9 +112,13 @@ * */ public class EsqlNodeSubclassTests> extends NodeSubclassTests { + private static final String ESQL_CORE_CLASS_PREFIX = "org.elasticsearch.xpack.esql.core"; + private static final String ESQL_CORE_JAR_LOCATION_SUBSTRING = "x-pack-esql-core"; + private static final String ESQL_CLASS_PREFIX = "org.elasticsearch.xpack.esql"; + private static final Predicate CLASSNAME_FILTER = className -> { - boolean esqlCore = className.startsWith("org.elasticsearch.xpack.esql.core") != false; - boolean esqlProper = className.startsWith("org.elasticsearch.xpack.esql") != false; + boolean esqlCore = className.startsWith(ESQL_CORE_CLASS_PREFIX) != false; + boolean esqlProper = className.startsWith(ESQL_CLASS_PREFIX) != false; return (esqlCore || esqlProper); }; @@ -164,15 +166,6 @@ public void testInfoParameters() throws Exception { */ expectedCount -= 1; - // special exceptions with private constructors - if (MetadataAttribute.class.equals(subclass) || ReferenceAttribute.class.equals(subclass)) { - expectedCount++; - } - - if (FieldAttribute.class.equals(subclass)) { - expectedCount += 2; - } - assertEquals(expectedCount, info(node).properties().size()); } @@ -736,7 +729,7 @@ public static Set> subclassesOf(Class clazz, Predicate // NIO FileSystem API is not used since it trips the SecurityManager // https://bugs.openjdk.java.net/browse/JDK-8160798 // so iterate the jar "by hand" - if (path.endsWith(".jar") && path.contains("x-pack-ql")) { + if (path.endsWith(".jar") && path.contains(ESQL_CORE_JAR_LOCATION_SUBSTRING)) { try (JarInputStream jar = jarStream(root)) { JarEntry je = null; while ((je = jar.getNextJarEntry()) != null) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index 632ba74f3b7b5..9d3a263b506c9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -36,6 +36,9 @@ public Set getFeatures() { @Override public Set getTestFeatures() { - return Set.of(SemanticTextFieldMapper.SEMANTIC_TEXT_IN_OBJECT_FIELD_FIX); + return Set.of( + SemanticTextFieldMapper.SEMANTIC_TEXT_IN_OBJECT_FIELD_FIX, + SemanticTextFieldMapper.SEMANTIC_TEXT_SINGLE_FIELD_UPDATE_FIX + ); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java index 1a6e4760fe125..b3bbe3a7df9bc 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java @@ -446,7 +446,8 @@ private Map> createFieldInferenceRequests(Bu String field = entry.getName(); String inferenceId = entry.getInferenceId(); var originalFieldValue = XContentMapValues.extractValue(field, docMap); - if (originalFieldValue instanceof Map) { + if (originalFieldValue instanceof Map || (originalFieldValue == null && entry.getSourceFields().length == 1)) { + // Inference has already been computed, or there is no inference required. continue; } int order = 0; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 9d5d465c4e794..d70931a85c82e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -91,6 +91,8 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie public static final NodeFeature SEMANTIC_TEXT_DEFAULT_ELSER_2 = new NodeFeature("semantic_text.default_elser_2"); public static final NodeFeature SEMANTIC_TEXT_IN_OBJECT_FIELD_FIX = new NodeFeature("semantic_text.in_object_field_fix"); + public static final NodeFeature SEMANTIC_TEXT_SINGLE_FIELD_UPDATE_FIX = new NodeFeature("semantic_text.single_field_update_fix"); + public static final String CONTENT_TYPE = "semantic_text"; public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID; diff --git a/x-pack/plugin/inference/src/yamlRestTest/java/org/elasticsearch/xpack/inference/InferenceRestIT.java b/x-pack/plugin/inference/src/yamlRestTest/java/org/elasticsearch/xpack/inference/InferenceRestIT.java index 701bcd204fcfe..fe406722ae1e2 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/java/org/elasticsearch/xpack/inference/InferenceRestIT.java +++ b/x-pack/plugin/inference/src/yamlRestTest/java/org/elasticsearch/xpack/inference/InferenceRestIT.java @@ -9,6 +9,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; @@ -30,6 +31,15 @@ public InferenceRestIT(final ClientYamlTestCandidate testCandidate) { super(testCandidate); } + @Override + protected Settings restClientSettings() { + var baseSettings = super.restClientSettings(); + return Settings.builder() + .put(baseSettings) + .put(CLIENT_SOCKET_TIMEOUT, "120s") // Long timeout for model download + .build(); + } + @Override protected String getTestRestCluster() { return cluster.getHttpAddresses(); diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml index 59ce439d954a2..294761608ee81 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/60_semantic_text_inference_update.yml @@ -610,3 +610,59 @@ setup: - exists: _source.dense_field.inference.chunks.0.embeddings - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" } - match: { _source.non_inference_field: "updated non inference test" } + +--- +"Bypass inference on bulk update operation": + - requires: + cluster_features: semantic_text.single_field_update_fix + reason: Standalone semantic text fields are now optional in a bulk update operation + + # Update as upsert + - do: + bulk: + body: + - '{"update": {"_index": "test-index", "_id": "doc_1"}}' + - '{"doc": { "sparse_field": "inference test", "dense_field": "another inference test", "non_inference_field": "non inference test" }, "doc_as_upsert": true}' + + - match: { errors: false } + - match: { items.0.update.result: "created" } + + - do: + bulk: + body: + - '{"update": {"_index": "test-index", "_id": "doc_1"}}' + - '{"doc": { "non_inference_field": "another value" }, "doc_as_upsert": true}' + + - match: { errors: false } + - match: { items.0.update.result: "updated" } + + - do: + get: + index: test-index + id: doc_1 + + - match: { _source.sparse_field.text: "inference test" } + - exists: _source.sparse_field.inference.chunks.0.embeddings + - match: { _source.sparse_field.inference.chunks.0.text: "inference test" } + - match: { _source.dense_field.text: "another inference test" } + - exists: _source.dense_field.inference.chunks.0.embeddings + - match: { _source.dense_field.inference.chunks.0.text: "another inference test" } + - match: { _source.non_inference_field: "another value" } + + - do: + bulk: + body: + - '{"update": {"_index": "test-index", "_id": "doc_1"}}' + - '{"doc": { "sparse_field": null, "dense_field": null, "non_inference_field": "updated value" }, "doc_as_upsert": true}' + + - match: { errors: false } + - match: { items.0.update.result: "updated" } + + - do: + get: + index: test-index + id: doc_1 + + - match: { _source.sparse_field: null } + - match: { _source.dense_field: null } + - match: { _source.non_inference_field: "updated value" } diff --git a/x-pack/plugin/kql/build.gradle b/x-pack/plugin/kql/build.gradle index 9d0860346b188..7e4df5654f225 100644 --- a/x-pack/plugin/kql/build.gradle +++ b/x-pack/plugin/kql/build.gradle @@ -1,8 +1,10 @@ import org.elasticsearch.gradle.internal.info.BuildParams + import static org.elasticsearch.gradle.util.PlatformUtils.normalize apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.publish' esplugin { @@ -17,19 +19,21 @@ base { dependencies { compileOnly project(path: xpackModule('core')) - api "org.antlr:antlr4-runtime:${versions.antlr4}" + implementation "org.antlr:antlr4-runtime:${versions.antlr4}" testImplementation "org.antlr:antlr4-runtime:${versions.antlr4}" testImplementation project(':test:framework') testImplementation(testArtifact(project(xpackModule('core')))) } -/**************************************************************** - * Enable QA/rest integration tests for snapshot builds only * - * TODO: Enable for all builds upon this feature release * - ****************************************************************/ -if (BuildParams.isSnapshotBuild()) { - addQaCheckDependencies(project) +tasks.named('yamlRestTest') { + usesDefaultDistribution() +}.configure { + /**************************************************************** + * Enable QA/rest integration tests for snapshot builds only * + * TODO: Enable for all builds upon this feature release * + ****************************************************************/ + enabled = BuildParams.isSnapshotBuild() } /********************************** diff --git a/x-pack/plugin/kql/src/main/antlr/KqlBase.g4 b/x-pack/plugin/kql/src/main/antlr/KqlBase.g4 index 52a70b9d4c018..da015b699cb15 100644 --- a/x-pack/plugin/kql/src/main/antlr/KqlBase.g4 +++ b/x-pack/plugin/kql/src/main/antlr/KqlBase.g4 @@ -88,7 +88,7 @@ fieldQueryValue ; fieldName - : value=UNQUOTED_LITERAL+ + : value=UNQUOTED_LITERAL | value=QUOTED_STRING | value=WILDCARD ; diff --git a/x-pack/plugin/kql/src/main/java/module-info.java b/x-pack/plugin/kql/src/main/java/module-info.java index c4dd539508f39..41e51033b9c70 100644 --- a/x-pack/plugin/kql/src/main/java/module-info.java +++ b/x-pack/plugin/kql/src/main/java/module-info.java @@ -16,4 +16,5 @@ exports org.elasticsearch.xpack.kql; exports org.elasticsearch.xpack.kql.parser; + exports org.elasticsearch.xpack.kql.query; } diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java index 4734924b23618..217513bd2c0da 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/KqlPlugin.java @@ -7,10 +7,21 @@ package org.elasticsearch.xpack.kql; +import org.elasticsearch.Build; import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.xpack.kql.query.KqlQueryBuilder; + +import java.util.List; public class KqlPlugin extends Plugin implements SearchPlugin, ExtensiblePlugin { + @Override + public List> getQueries() { + if (Build.current().isSnapshot()) { + return List.of(new SearchPlugin.QuerySpec<>(KqlQueryBuilder.NAME, KqlQueryBuilder::new, KqlQueryBuilder::fromXContent)); + } + return List.of(); + } } diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java index 5f1106522e47f..67d7da0381d67 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlAstBuilder.java @@ -9,6 +9,7 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -16,29 +17,34 @@ import org.elasticsearch.index.query.MultiMatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; +import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BiFunction; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; -import static org.elasticsearch.xpack.kql.parser.KqlParserExecutionContext.isDateField; -import static org.elasticsearch.xpack.kql.parser.KqlParserExecutionContext.isKeywordField; -import static org.elasticsearch.xpack.kql.parser.KqlParserExecutionContext.isRuntimeField; +import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isDateField; +import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isKeywordField; +import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isRuntimeField; +import static org.elasticsearch.xpack.kql.parser.KqlParsingContext.isSearchableField; import static org.elasticsearch.xpack.kql.parser.ParserUtils.escapeLuceneQueryString; +import static org.elasticsearch.xpack.kql.parser.ParserUtils.extractText; import static org.elasticsearch.xpack.kql.parser.ParserUtils.hasWildcard; +import static org.elasticsearch.xpack.kql.parser.ParserUtils.typedParsing; class KqlAstBuilder extends KqlBaseBaseVisitor { - private final KqlParserExecutionContext kqlParserExecutionContext; + private final KqlParsingContext kqlParsingContext; - KqlAstBuilder(KqlParserExecutionContext kqlParserExecutionContext) { - this.kqlParserExecutionContext = kqlParserExecutionContext; + KqlAstBuilder(KqlParsingContext kqlParsingContext) { + this.kqlParsingContext = kqlParsingContext; } public QueryBuilder toQueryBuilder(ParserRuleContext ctx) { if (ctx instanceof KqlBaseParser.TopLevelQueryContext topLeveQueryContext) { if (topLeveQueryContext.query() != null) { - return ParserUtils.typedParsing(this, topLeveQueryContext.query(), QueryBuilder.class); + return typedParsing(this, topLeveQueryContext.query(), QueryBuilder.class); } return new MatchAllQueryBuilder(); @@ -59,9 +65,9 @@ public QueryBuilder visitAndBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) // TODO: KQLContext has an option to wrap the clauses into a filter instead of a must clause. Do we need it? for (ParserRuleContext subQueryCtx : ctx.query()) { if (subQueryCtx instanceof KqlBaseParser.BooleanQueryContext booleanSubQueryCtx && isAndQuery(booleanSubQueryCtx)) { - ParserUtils.typedParsing(this, subQueryCtx, BoolQueryBuilder.class).must().forEach(builder::must); + typedParsing(this, subQueryCtx, BoolQueryBuilder.class).must().forEach(builder::must); } else { - builder.must(ParserUtils.typedParsing(this, subQueryCtx, QueryBuilder.class)); + builder.must(typedParsing(this, subQueryCtx, QueryBuilder.class)); } } @@ -73,9 +79,9 @@ public QueryBuilder visitOrBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) { for (ParserRuleContext subQueryCtx : ctx.query()) { if (subQueryCtx instanceof KqlBaseParser.BooleanQueryContext booleanSubQueryCtx && isOrQuery(booleanSubQueryCtx)) { - ParserUtils.typedParsing(this, subQueryCtx, BoolQueryBuilder.class).should().forEach(builder::should); + typedParsing(this, subQueryCtx, BoolQueryBuilder.class).should().forEach(builder::should); } else { - builder.should(ParserUtils.typedParsing(this, subQueryCtx, QueryBuilder.class)); + builder.should(typedParsing(this, subQueryCtx, QueryBuilder.class)); } } @@ -84,12 +90,12 @@ public QueryBuilder visitOrBooleanQuery(KqlBaseParser.BooleanQueryContext ctx) { @Override public QueryBuilder visitNotQuery(KqlBaseParser.NotQueryContext ctx) { - return QueryBuilders.boolQuery().mustNot(ParserUtils.typedParsing(this, ctx.simpleQuery(), QueryBuilder.class)); + return QueryBuilders.boolQuery().mustNot(typedParsing(this, ctx.simpleQuery(), QueryBuilder.class)); } @Override public QueryBuilder visitParenthesizedQuery(KqlBaseParser.ParenthesizedQueryContext ctx) { - return ParserUtils.typedParsing(this, ctx.query(), QueryBuilder.class); + return typedParsing(this, ctx.query(), QueryBuilder.class); } @Override @@ -121,12 +127,16 @@ public QueryBuilder visitExistsQuery(KqlBaseParser.ExistsQueryContext ctx) { public QueryBuilder visitRangeQuery(KqlBaseParser.RangeQueryContext ctx) { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1); - String queryText = ParserUtils.extractText(ctx.rangeQueryValue()); + String queryText = extractText(ctx.rangeQueryValue()); BiFunction rangeOperation = rangeOperation(ctx.operator); withFields(ctx.fieldName(), (fieldName, mappedFieldType) -> { RangeQueryBuilder rangeQuery = rangeOperation.apply(QueryBuilders.rangeQuery(fieldName), queryText); - // TODO: add timezone for date fields + + if (kqlParsingContext.timeZone() != null) { + rangeQuery.timeZone(kqlParsingContext.timeZone().getId()); + } + boolQueryBuilder.should(rangeQuery); }); @@ -135,42 +145,54 @@ public QueryBuilder visitRangeQuery(KqlBaseParser.RangeQueryContext ctx) { @Override public QueryBuilder visitFieldLessQuery(KqlBaseParser.FieldLessQueryContext ctx) { - String queryText = ParserUtils.extractText(ctx.fieldQueryValue()); + String queryText = extractText(ctx.fieldQueryValue()); if (hasWildcard(ctx.fieldQueryValue())) { - // TODO: set default fields. - return QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)); + QueryStringQueryBuilder queryString = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)); + if (kqlParsingContext.defaultField() != null) { + queryString.defaultField(kqlParsingContext.defaultField()); + } + return queryString; } boolean isPhraseMatch = ctx.fieldQueryValue().QUOTED_STRING() != null; - return QueryBuilders.multiMatchQuery(queryText) - // TODO: add default fields? + MultiMatchQueryBuilder multiMatchQuery = QueryBuilders.multiMatchQuery(queryText) .type(isPhraseMatch ? MultiMatchQueryBuilder.Type.PHRASE : MultiMatchQueryBuilder.Type.BEST_FIELDS) .lenient(true); + + if (kqlParsingContext.defaultField() != null) { + kqlParsingContext.resolveDefaultFieldNames() + .stream() + .filter(kqlParsingContext::isSearchableField) + .forEach(multiMatchQuery::field); + } + + return multiMatchQuery; } @Override public QueryBuilder visitFieldQuery(KqlBaseParser.FieldQueryContext ctx) { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1); - String queryText = ParserUtils.extractText(ctx.fieldQueryValue()); + String queryText = extractText(ctx.fieldQueryValue()); boolean hasWildcard = hasWildcard(ctx.fieldQueryValue()); withFields(ctx.fieldName(), (fieldName, mappedFieldType) -> { QueryBuilder fieldQuery = null; if (hasWildcard && isKeywordField(mappedFieldType)) { - fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText) - .caseInsensitive(kqlParserExecutionContext.isCaseSensitive() == false); + fieldQuery = QueryBuilders.wildcardQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive()); } else if (hasWildcard) { fieldQuery = QueryBuilders.queryStringQuery(escapeLuceneQueryString(queryText, true)).field(fieldName); } else if (isDateField(mappedFieldType)) { - // TODO: add timezone - fieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText); + RangeQueryBuilder rangeFieldQuery = QueryBuilders.rangeQuery(fieldName).gte(queryText).lte(queryText); + if (kqlParsingContext.timeZone() != null) { + rangeFieldQuery.timeZone(kqlParsingContext.timeZone().getId()); + } + fieldQuery = rangeFieldQuery; } else if (isKeywordField(mappedFieldType)) { - fieldQuery = QueryBuilders.termQuery(fieldName, queryText) - .caseInsensitive(kqlParserExecutionContext.isCaseSensitive() == false); + fieldQuery = QueryBuilders.termQuery(fieldName, queryText).caseInsensitive(kqlParsingContext.caseInsensitive()); } else if (ctx.fieldQueryValue().QUOTED_STRING() != null) { fieldQuery = QueryBuilders.matchPhraseQuery(fieldName, queryText); } else { @@ -194,7 +216,26 @@ private static boolean isOrQuery(KqlBaseParser.BooleanQueryContext ctx) { } private void withFields(KqlBaseParser.FieldNameContext ctx, BiConsumer fieldConsummer) { - kqlParserExecutionContext.resolveFields(ctx).forEach(fieldDef -> fieldConsummer.accept(fieldDef.v1(), fieldDef.v2())); + assert ctx != null : "Field ctx cannot be null"; + String fieldNamePattern = extractText(ctx); + Set fieldNames = kqlParsingContext.resolveFieldNames(fieldNamePattern); + + if (ctx.value.getType() == KqlBaseParser.QUOTED_STRING && Regex.isSimpleMatchPattern(fieldNamePattern)) { + // When using quoted string, wildcards are not expanded. + // No field can match and we can return early. + return; + } + + if (ctx.value.getType() == KqlBaseParser.QUOTED_STRING) { + assert fieldNames.size() < 2 : "expecting only one matching field"; + } + + fieldNames.forEach(fieldName -> { + MappedFieldType fieldType = kqlParsingContext.fieldType(fieldName); + if (isSearchableField(fieldName, fieldType)) { + fieldConsummer.accept(fieldName, fieldType); + } + }); } private QueryBuilder rewriteDisjunctionQuery(BoolQueryBuilder boolQueryBuilder) { diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp index 2b09dd52e95b0..7af37d7e3c3b5 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBase.interp @@ -54,4 +54,4 @@ fieldName atn: -[4, 1, 16, 140, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 1, 0, 3, 0, 30, 8, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 40, 8, 1, 10, 1, 12, 1, 43, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 53, 8, 2, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 3, 5, 66, 8, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 4, 8, 79, 8, 8, 11, 8, 12, 8, 80, 1, 8, 3, 8, 84, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 100, 8, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 3, 11, 107, 8, 11, 1, 12, 3, 12, 110, 8, 12, 1, 12, 4, 12, 113, 8, 12, 11, 12, 12, 12, 114, 1, 12, 3, 12, 118, 8, 12, 1, 12, 1, 12, 3, 12, 122, 8, 12, 1, 12, 1, 12, 3, 12, 126, 8, 12, 1, 12, 3, 12, 129, 8, 12, 1, 13, 4, 13, 132, 8, 13, 11, 13, 12, 13, 133, 1, 13, 1, 13, 3, 13, 138, 8, 13, 1, 13, 0, 1, 2, 14, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 0, 4, 1, 0, 2, 3, 1, 0, 6, 9, 2, 0, 14, 14, 16, 16, 1, 0, 2, 4, 150, 0, 29, 1, 0, 0, 0, 2, 33, 1, 0, 0, 0, 4, 52, 1, 0, 0, 0, 6, 54, 1, 0, 0, 0, 8, 57, 1, 0, 0, 0, 10, 65, 1, 0, 0, 0, 12, 69, 1, 0, 0, 0, 14, 73, 1, 0, 0, 0, 16, 83, 1, 0, 0, 0, 18, 85, 1, 0, 0, 0, 20, 99, 1, 0, 0, 0, 22, 106, 1, 0, 0, 0, 24, 128, 1, 0, 0, 0, 26, 137, 1, 0, 0, 0, 28, 30, 3, 2, 1, 0, 29, 28, 1, 0, 0, 0, 29, 30, 1, 0, 0, 0, 30, 31, 1, 0, 0, 0, 31, 32, 5, 0, 0, 1, 32, 1, 1, 0, 0, 0, 33, 34, 6, 1, -1, 0, 34, 35, 3, 4, 2, 0, 35, 41, 1, 0, 0, 0, 36, 37, 10, 2, 0, 0, 37, 38, 7, 0, 0, 0, 38, 40, 3, 2, 1, 2, 39, 36, 1, 0, 0, 0, 40, 43, 1, 0, 0, 0, 41, 39, 1, 0, 0, 0, 41, 42, 1, 0, 0, 0, 42, 3, 1, 0, 0, 0, 43, 41, 1, 0, 0, 0, 44, 53, 3, 6, 3, 0, 45, 53, 3, 8, 4, 0, 46, 53, 3, 12, 6, 0, 47, 53, 3, 10, 5, 0, 48, 53, 3, 18, 9, 0, 49, 53, 3, 14, 7, 0, 50, 53, 3, 20, 10, 0, 51, 53, 3, 22, 11, 0, 52, 44, 1, 0, 0, 0, 52, 45, 1, 0, 0, 0, 52, 46, 1, 0, 0, 0, 52, 47, 1, 0, 0, 0, 52, 48, 1, 0, 0, 0, 52, 49, 1, 0, 0, 0, 52, 50, 1, 0, 0, 0, 52, 51, 1, 0, 0, 0, 53, 5, 1, 0, 0, 0, 54, 55, 5, 4, 0, 0, 55, 56, 3, 4, 2, 0, 56, 7, 1, 0, 0, 0, 57, 58, 3, 26, 13, 0, 58, 59, 5, 5, 0, 0, 59, 60, 5, 12, 0, 0, 60, 61, 3, 2, 1, 0, 61, 62, 5, 13, 0, 0, 62, 9, 1, 0, 0, 0, 63, 64, 5, 16, 0, 0, 64, 66, 5, 5, 0, 0, 65, 63, 1, 0, 0, 0, 65, 66, 1, 0, 0, 0, 66, 67, 1, 0, 0, 0, 67, 68, 5, 16, 0, 0, 68, 11, 1, 0, 0, 0, 69, 70, 5, 10, 0, 0, 70, 71, 3, 2, 1, 0, 71, 72, 5, 11, 0, 0, 72, 13, 1, 0, 0, 0, 73, 74, 3, 26, 13, 0, 74, 75, 7, 1, 0, 0, 75, 76, 3, 16, 8, 0, 76, 15, 1, 0, 0, 0, 77, 79, 7, 2, 0, 0, 78, 77, 1, 0, 0, 0, 79, 80, 1, 0, 0, 0, 80, 78, 1, 0, 0, 0, 80, 81, 1, 0, 0, 0, 81, 84, 1, 0, 0, 0, 82, 84, 5, 15, 0, 0, 83, 78, 1, 0, 0, 0, 83, 82, 1, 0, 0, 0, 84, 17, 1, 0, 0, 0, 85, 86, 3, 26, 13, 0, 86, 87, 5, 5, 0, 0, 87, 88, 5, 16, 0, 0, 88, 19, 1, 0, 0, 0, 89, 90, 3, 26, 13, 0, 90, 91, 5, 5, 0, 0, 91, 92, 3, 24, 12, 0, 92, 100, 1, 0, 0, 0, 93, 94, 3, 26, 13, 0, 94, 95, 5, 5, 0, 0, 95, 96, 5, 10, 0, 0, 96, 97, 3, 24, 12, 0, 97, 98, 5, 11, 0, 0, 98, 100, 1, 0, 0, 0, 99, 89, 1, 0, 0, 0, 99, 93, 1, 0, 0, 0, 100, 21, 1, 0, 0, 0, 101, 107, 3, 24, 12, 0, 102, 103, 5, 10, 0, 0, 103, 104, 3, 24, 12, 0, 104, 105, 5, 11, 0, 0, 105, 107, 1, 0, 0, 0, 106, 101, 1, 0, 0, 0, 106, 102, 1, 0, 0, 0, 107, 23, 1, 0, 0, 0, 108, 110, 7, 3, 0, 0, 109, 108, 1, 0, 0, 0, 109, 110, 1, 0, 0, 0, 110, 112, 1, 0, 0, 0, 111, 113, 7, 2, 0, 0, 112, 111, 1, 0, 0, 0, 113, 114, 1, 0, 0, 0, 114, 112, 1, 0, 0, 0, 114, 115, 1, 0, 0, 0, 115, 117, 1, 0, 0, 0, 116, 118, 7, 3, 0, 0, 117, 116, 1, 0, 0, 0, 117, 118, 1, 0, 0, 0, 118, 129, 1, 0, 0, 0, 119, 121, 7, 0, 0, 0, 120, 122, 7, 3, 0, 0, 121, 120, 1, 0, 0, 0, 121, 122, 1, 0, 0, 0, 122, 129, 1, 0, 0, 0, 123, 125, 5, 4, 0, 0, 124, 126, 7, 0, 0, 0, 125, 124, 1, 0, 0, 0, 125, 126, 1, 0, 0, 0, 126, 129, 1, 0, 0, 0, 127, 129, 5, 15, 0, 0, 128, 109, 1, 0, 0, 0, 128, 119, 1, 0, 0, 0, 128, 123, 1, 0, 0, 0, 128, 127, 1, 0, 0, 0, 129, 25, 1, 0, 0, 0, 130, 132, 5, 14, 0, 0, 131, 130, 1, 0, 0, 0, 132, 133, 1, 0, 0, 0, 133, 131, 1, 0, 0, 0, 133, 134, 1, 0, 0, 0, 134, 138, 1, 0, 0, 0, 135, 138, 5, 15, 0, 0, 136, 138, 5, 16, 0, 0, 137, 131, 1, 0, 0, 0, 137, 135, 1, 0, 0, 0, 137, 136, 1, 0, 0, 0, 138, 27, 1, 0, 0, 0, 16, 29, 41, 52, 65, 80, 83, 99, 106, 109, 114, 117, 121, 125, 128, 133, 137] \ No newline at end of file +[4, 1, 16, 136, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 1, 0, 3, 0, 30, 8, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 40, 8, 1, 10, 1, 12, 1, 43, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 53, 8, 2, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 3, 5, 66, 8, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 4, 8, 79, 8, 8, 11, 8, 12, 8, 80, 1, 8, 3, 8, 84, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 100, 8, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 3, 11, 107, 8, 11, 1, 12, 3, 12, 110, 8, 12, 1, 12, 4, 12, 113, 8, 12, 11, 12, 12, 12, 114, 1, 12, 3, 12, 118, 8, 12, 1, 12, 1, 12, 3, 12, 122, 8, 12, 1, 12, 1, 12, 3, 12, 126, 8, 12, 1, 12, 3, 12, 129, 8, 12, 1, 13, 1, 13, 1, 13, 3, 13, 134, 8, 13, 1, 13, 0, 1, 2, 14, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 0, 4, 1, 0, 2, 3, 1, 0, 6, 9, 2, 0, 14, 14, 16, 16, 1, 0, 2, 4, 145, 0, 29, 1, 0, 0, 0, 2, 33, 1, 0, 0, 0, 4, 52, 1, 0, 0, 0, 6, 54, 1, 0, 0, 0, 8, 57, 1, 0, 0, 0, 10, 65, 1, 0, 0, 0, 12, 69, 1, 0, 0, 0, 14, 73, 1, 0, 0, 0, 16, 83, 1, 0, 0, 0, 18, 85, 1, 0, 0, 0, 20, 99, 1, 0, 0, 0, 22, 106, 1, 0, 0, 0, 24, 128, 1, 0, 0, 0, 26, 133, 1, 0, 0, 0, 28, 30, 3, 2, 1, 0, 29, 28, 1, 0, 0, 0, 29, 30, 1, 0, 0, 0, 30, 31, 1, 0, 0, 0, 31, 32, 5, 0, 0, 1, 32, 1, 1, 0, 0, 0, 33, 34, 6, 1, -1, 0, 34, 35, 3, 4, 2, 0, 35, 41, 1, 0, 0, 0, 36, 37, 10, 2, 0, 0, 37, 38, 7, 0, 0, 0, 38, 40, 3, 2, 1, 2, 39, 36, 1, 0, 0, 0, 40, 43, 1, 0, 0, 0, 41, 39, 1, 0, 0, 0, 41, 42, 1, 0, 0, 0, 42, 3, 1, 0, 0, 0, 43, 41, 1, 0, 0, 0, 44, 53, 3, 6, 3, 0, 45, 53, 3, 8, 4, 0, 46, 53, 3, 12, 6, 0, 47, 53, 3, 10, 5, 0, 48, 53, 3, 18, 9, 0, 49, 53, 3, 14, 7, 0, 50, 53, 3, 20, 10, 0, 51, 53, 3, 22, 11, 0, 52, 44, 1, 0, 0, 0, 52, 45, 1, 0, 0, 0, 52, 46, 1, 0, 0, 0, 52, 47, 1, 0, 0, 0, 52, 48, 1, 0, 0, 0, 52, 49, 1, 0, 0, 0, 52, 50, 1, 0, 0, 0, 52, 51, 1, 0, 0, 0, 53, 5, 1, 0, 0, 0, 54, 55, 5, 4, 0, 0, 55, 56, 3, 4, 2, 0, 56, 7, 1, 0, 0, 0, 57, 58, 3, 26, 13, 0, 58, 59, 5, 5, 0, 0, 59, 60, 5, 12, 0, 0, 60, 61, 3, 2, 1, 0, 61, 62, 5, 13, 0, 0, 62, 9, 1, 0, 0, 0, 63, 64, 5, 16, 0, 0, 64, 66, 5, 5, 0, 0, 65, 63, 1, 0, 0, 0, 65, 66, 1, 0, 0, 0, 66, 67, 1, 0, 0, 0, 67, 68, 5, 16, 0, 0, 68, 11, 1, 0, 0, 0, 69, 70, 5, 10, 0, 0, 70, 71, 3, 2, 1, 0, 71, 72, 5, 11, 0, 0, 72, 13, 1, 0, 0, 0, 73, 74, 3, 26, 13, 0, 74, 75, 7, 1, 0, 0, 75, 76, 3, 16, 8, 0, 76, 15, 1, 0, 0, 0, 77, 79, 7, 2, 0, 0, 78, 77, 1, 0, 0, 0, 79, 80, 1, 0, 0, 0, 80, 78, 1, 0, 0, 0, 80, 81, 1, 0, 0, 0, 81, 84, 1, 0, 0, 0, 82, 84, 5, 15, 0, 0, 83, 78, 1, 0, 0, 0, 83, 82, 1, 0, 0, 0, 84, 17, 1, 0, 0, 0, 85, 86, 3, 26, 13, 0, 86, 87, 5, 5, 0, 0, 87, 88, 5, 16, 0, 0, 88, 19, 1, 0, 0, 0, 89, 90, 3, 26, 13, 0, 90, 91, 5, 5, 0, 0, 91, 92, 3, 24, 12, 0, 92, 100, 1, 0, 0, 0, 93, 94, 3, 26, 13, 0, 94, 95, 5, 5, 0, 0, 95, 96, 5, 10, 0, 0, 96, 97, 3, 24, 12, 0, 97, 98, 5, 11, 0, 0, 98, 100, 1, 0, 0, 0, 99, 89, 1, 0, 0, 0, 99, 93, 1, 0, 0, 0, 100, 21, 1, 0, 0, 0, 101, 107, 3, 24, 12, 0, 102, 103, 5, 10, 0, 0, 103, 104, 3, 24, 12, 0, 104, 105, 5, 11, 0, 0, 105, 107, 1, 0, 0, 0, 106, 101, 1, 0, 0, 0, 106, 102, 1, 0, 0, 0, 107, 23, 1, 0, 0, 0, 108, 110, 7, 3, 0, 0, 109, 108, 1, 0, 0, 0, 109, 110, 1, 0, 0, 0, 110, 112, 1, 0, 0, 0, 111, 113, 7, 2, 0, 0, 112, 111, 1, 0, 0, 0, 113, 114, 1, 0, 0, 0, 114, 112, 1, 0, 0, 0, 114, 115, 1, 0, 0, 0, 115, 117, 1, 0, 0, 0, 116, 118, 7, 3, 0, 0, 117, 116, 1, 0, 0, 0, 117, 118, 1, 0, 0, 0, 118, 129, 1, 0, 0, 0, 119, 121, 7, 0, 0, 0, 120, 122, 7, 3, 0, 0, 121, 120, 1, 0, 0, 0, 121, 122, 1, 0, 0, 0, 122, 129, 1, 0, 0, 0, 123, 125, 5, 4, 0, 0, 124, 126, 7, 0, 0, 0, 125, 124, 1, 0, 0, 0, 125, 126, 1, 0, 0, 0, 126, 129, 1, 0, 0, 0, 127, 129, 5, 15, 0, 0, 128, 109, 1, 0, 0, 0, 128, 119, 1, 0, 0, 0, 128, 123, 1, 0, 0, 0, 128, 127, 1, 0, 0, 0, 129, 25, 1, 0, 0, 0, 130, 134, 5, 14, 0, 0, 131, 134, 5, 15, 0, 0, 132, 134, 5, 16, 0, 0, 133, 130, 1, 0, 0, 0, 133, 131, 1, 0, 0, 0, 133, 132, 1, 0, 0, 0, 134, 27, 1, 0, 0, 0, 15, 29, 41, 52, 65, 80, 83, 99, 106, 109, 114, 117, 121, 125, 128, 133] \ No newline at end of file diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java index b4b0a69a82387..118ac32aadd61 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlBaseParser.java @@ -1194,10 +1194,7 @@ public final FieldQueryValueContext fieldQueryValue() throws RecognitionExceptio @SuppressWarnings("CheckReturnValue") public static class FieldNameContext extends ParserRuleContext { public Token value; - public List UNQUOTED_LITERAL() { return getTokens(KqlBaseParser.UNQUOTED_LITERAL); } - public TerminalNode UNQUOTED_LITERAL(int i) { - return getToken(KqlBaseParser.UNQUOTED_LITERAL, i); - } + public TerminalNode UNQUOTED_LITERAL() { return getToken(KqlBaseParser.UNQUOTED_LITERAL, 0); } public TerminalNode QUOTED_STRING() { return getToken(KqlBaseParser.QUOTED_STRING, 0); } public TerminalNode WILDCARD() { return getToken(KqlBaseParser.WILDCARD, 0); } public FieldNameContext(ParserRuleContext parent, int invokingState) { @@ -1222,41 +1219,28 @@ public T accept(ParseTreeVisitor visitor) { public final FieldNameContext fieldName() throws RecognitionException { FieldNameContext _localctx = new FieldNameContext(_ctx, getState()); enterRule(_localctx, 26, RULE_fieldName); - int _la; try { - setState(137); + setState(133); _errHandler.sync(this); switch (_input.LA(1)) { case UNQUOTED_LITERAL: enterOuterAlt(_localctx, 1); { - setState(131); - _errHandler.sync(this); - _la = _input.LA(1); - do { - { - { - setState(130); - ((FieldNameContext)_localctx).value = match(UNQUOTED_LITERAL); - } - } - setState(133); - _errHandler.sync(this); - _la = _input.LA(1); - } while ( _la==UNQUOTED_LITERAL ); + setState(130); + ((FieldNameContext)_localctx).value = match(UNQUOTED_LITERAL); } break; case QUOTED_STRING: enterOuterAlt(_localctx, 2); { - setState(135); + setState(131); ((FieldNameContext)_localctx).value = match(QUOTED_STRING); } break; case WILDCARD: enterOuterAlt(_localctx, 3); { - setState(136); + setState(132); ((FieldNameContext)_localctx).value = match(WILDCARD); } break; @@ -1291,7 +1275,7 @@ private boolean query_sempred(QueryContext _localctx, int predIndex) { } public static final String _serializedATN = - "\u0004\u0001\u0010\u008c\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ + "\u0004\u0001\u0010\u0088\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ "\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004"+ "\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007"+ "\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b"+ @@ -1309,70 +1293,67 @@ private boolean query_sempred(QueryContext _localctx, int predIndex) { "\nd\b\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0003"+ "\u000bk\b\u000b\u0001\f\u0003\fn\b\f\u0001\f\u0004\fq\b\f\u000b\f\f\f"+ "r\u0001\f\u0003\fv\b\f\u0001\f\u0001\f\u0003\fz\b\f\u0001\f\u0001\f\u0003"+ - "\f~\b\f\u0001\f\u0003\f\u0081\b\f\u0001\r\u0004\r\u0084\b\r\u000b\r\f"+ - "\r\u0085\u0001\r\u0001\r\u0003\r\u008a\b\r\u0001\r\u0000\u0001\u0002\u000e"+ - "\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a"+ - "\u0000\u0004\u0001\u0000\u0002\u0003\u0001\u0000\u0006\t\u0002\u0000\u000e"+ - "\u000e\u0010\u0010\u0001\u0000\u0002\u0004\u0096\u0000\u001d\u0001\u0000"+ - "\u0000\u0000\u0002!\u0001\u0000\u0000\u0000\u00044\u0001\u0000\u0000\u0000"+ - "\u00066\u0001\u0000\u0000\u0000\b9\u0001\u0000\u0000\u0000\nA\u0001\u0000"+ - "\u0000\u0000\fE\u0001\u0000\u0000\u0000\u000eI\u0001\u0000\u0000\u0000"+ - "\u0010S\u0001\u0000\u0000\u0000\u0012U\u0001\u0000\u0000\u0000\u0014c"+ - "\u0001\u0000\u0000\u0000\u0016j\u0001\u0000\u0000\u0000\u0018\u0080\u0001"+ - "\u0000\u0000\u0000\u001a\u0089\u0001\u0000\u0000\u0000\u001c\u001e\u0003"+ - "\u0002\u0001\u0000\u001d\u001c\u0001\u0000\u0000\u0000\u001d\u001e\u0001"+ - "\u0000\u0000\u0000\u001e\u001f\u0001\u0000\u0000\u0000\u001f \u0005\u0000"+ - "\u0000\u0001 \u0001\u0001\u0000\u0000\u0000!\"\u0006\u0001\uffff\uffff"+ - "\u0000\"#\u0003\u0004\u0002\u0000#)\u0001\u0000\u0000\u0000$%\n\u0002"+ - "\u0000\u0000%&\u0007\u0000\u0000\u0000&(\u0003\u0002\u0001\u0002\'$\u0001"+ - "\u0000\u0000\u0000(+\u0001\u0000\u0000\u0000)\'\u0001\u0000\u0000\u0000"+ - ")*\u0001\u0000\u0000\u0000*\u0003\u0001\u0000\u0000\u0000+)\u0001\u0000"+ - "\u0000\u0000,5\u0003\u0006\u0003\u0000-5\u0003\b\u0004\u0000.5\u0003\f"+ - "\u0006\u0000/5\u0003\n\u0005\u000005\u0003\u0012\t\u000015\u0003\u000e"+ - "\u0007\u000025\u0003\u0014\n\u000035\u0003\u0016\u000b\u00004,\u0001\u0000"+ - "\u0000\u00004-\u0001\u0000\u0000\u00004.\u0001\u0000\u0000\u00004/\u0001"+ - "\u0000\u0000\u000040\u0001\u0000\u0000\u000041\u0001\u0000\u0000\u0000"+ - "42\u0001\u0000\u0000\u000043\u0001\u0000\u0000\u00005\u0005\u0001\u0000"+ - "\u0000\u000067\u0005\u0004\u0000\u000078\u0003\u0004\u0002\u00008\u0007"+ - "\u0001\u0000\u0000\u00009:\u0003\u001a\r\u0000:;\u0005\u0005\u0000\u0000"+ - ";<\u0005\f\u0000\u0000<=\u0003\u0002\u0001\u0000=>\u0005\r\u0000\u0000"+ - ">\t\u0001\u0000\u0000\u0000?@\u0005\u0010\u0000\u0000@B\u0005\u0005\u0000"+ - "\u0000A?\u0001\u0000\u0000\u0000AB\u0001\u0000\u0000\u0000BC\u0001\u0000"+ - "\u0000\u0000CD\u0005\u0010\u0000\u0000D\u000b\u0001\u0000\u0000\u0000"+ - "EF\u0005\n\u0000\u0000FG\u0003\u0002\u0001\u0000GH\u0005\u000b\u0000\u0000"+ - "H\r\u0001\u0000\u0000\u0000IJ\u0003\u001a\r\u0000JK\u0007\u0001\u0000"+ - "\u0000KL\u0003\u0010\b\u0000L\u000f\u0001\u0000\u0000\u0000MO\u0007\u0002"+ - "\u0000\u0000NM\u0001\u0000\u0000\u0000OP\u0001\u0000\u0000\u0000PN\u0001"+ - "\u0000\u0000\u0000PQ\u0001\u0000\u0000\u0000QT\u0001\u0000\u0000\u0000"+ - "RT\u0005\u000f\u0000\u0000SN\u0001\u0000\u0000\u0000SR\u0001\u0000\u0000"+ - "\u0000T\u0011\u0001\u0000\u0000\u0000UV\u0003\u001a\r\u0000VW\u0005\u0005"+ - "\u0000\u0000WX\u0005\u0010\u0000\u0000X\u0013\u0001\u0000\u0000\u0000"+ - "YZ\u0003\u001a\r\u0000Z[\u0005\u0005\u0000\u0000[\\\u0003\u0018\f\u0000"+ - "\\d\u0001\u0000\u0000\u0000]^\u0003\u001a\r\u0000^_\u0005\u0005\u0000"+ - "\u0000_`\u0005\n\u0000\u0000`a\u0003\u0018\f\u0000ab\u0005\u000b\u0000"+ - "\u0000bd\u0001\u0000\u0000\u0000cY\u0001\u0000\u0000\u0000c]\u0001\u0000"+ - "\u0000\u0000d\u0015\u0001\u0000\u0000\u0000ek\u0003\u0018\f\u0000fg\u0005"+ - "\n\u0000\u0000gh\u0003\u0018\f\u0000hi\u0005\u000b\u0000\u0000ik\u0001"+ - "\u0000\u0000\u0000je\u0001\u0000\u0000\u0000jf\u0001\u0000\u0000\u0000"+ - "k\u0017\u0001\u0000\u0000\u0000ln\u0007\u0003\u0000\u0000ml\u0001\u0000"+ - "\u0000\u0000mn\u0001\u0000\u0000\u0000np\u0001\u0000\u0000\u0000oq\u0007"+ - "\u0002\u0000\u0000po\u0001\u0000\u0000\u0000qr\u0001\u0000\u0000\u0000"+ - "rp\u0001\u0000\u0000\u0000rs\u0001\u0000\u0000\u0000su\u0001\u0000\u0000"+ - "\u0000tv\u0007\u0003\u0000\u0000ut\u0001\u0000\u0000\u0000uv\u0001\u0000"+ - "\u0000\u0000v\u0081\u0001\u0000\u0000\u0000wy\u0007\u0000\u0000\u0000"+ - "xz\u0007\u0003\u0000\u0000yx\u0001\u0000\u0000\u0000yz\u0001\u0000\u0000"+ - "\u0000z\u0081\u0001\u0000\u0000\u0000{}\u0005\u0004\u0000\u0000|~\u0007"+ - "\u0000\u0000\u0000}|\u0001\u0000\u0000\u0000}~\u0001\u0000\u0000\u0000"+ - "~\u0081\u0001\u0000\u0000\u0000\u007f\u0081\u0005\u000f\u0000\u0000\u0080"+ - "m\u0001\u0000\u0000\u0000\u0080w\u0001\u0000\u0000\u0000\u0080{\u0001"+ - "\u0000\u0000\u0000\u0080\u007f\u0001\u0000\u0000\u0000\u0081\u0019\u0001"+ - "\u0000\u0000\u0000\u0082\u0084\u0005\u000e\u0000\u0000\u0083\u0082\u0001"+ - "\u0000\u0000\u0000\u0084\u0085\u0001\u0000\u0000\u0000\u0085\u0083\u0001"+ - "\u0000\u0000\u0000\u0085\u0086\u0001\u0000\u0000\u0000\u0086\u008a\u0001"+ - "\u0000\u0000\u0000\u0087\u008a\u0005\u000f\u0000\u0000\u0088\u008a\u0005"+ - "\u0010\u0000\u0000\u0089\u0083\u0001\u0000\u0000\u0000\u0089\u0087\u0001"+ - "\u0000\u0000\u0000\u0089\u0088\u0001\u0000\u0000\u0000\u008a\u001b\u0001"+ - "\u0000\u0000\u0000\u0010\u001d)4APScjmruy}\u0080\u0085\u0089"; + "\f~\b\f\u0001\f\u0003\f\u0081\b\f\u0001\r\u0001\r\u0001\r\u0003\r\u0086"+ + "\b\r\u0001\r\u0000\u0001\u0002\u000e\u0000\u0002\u0004\u0006\b\n\f\u000e"+ + "\u0010\u0012\u0014\u0016\u0018\u001a\u0000\u0004\u0001\u0000\u0002\u0003"+ + "\u0001\u0000\u0006\t\u0002\u0000\u000e\u000e\u0010\u0010\u0001\u0000\u0002"+ + "\u0004\u0091\u0000\u001d\u0001\u0000\u0000\u0000\u0002!\u0001\u0000\u0000"+ + "\u0000\u00044\u0001\u0000\u0000\u0000\u00066\u0001\u0000\u0000\u0000\b"+ + "9\u0001\u0000\u0000\u0000\nA\u0001\u0000\u0000\u0000\fE\u0001\u0000\u0000"+ + "\u0000\u000eI\u0001\u0000\u0000\u0000\u0010S\u0001\u0000\u0000\u0000\u0012"+ + "U\u0001\u0000\u0000\u0000\u0014c\u0001\u0000\u0000\u0000\u0016j\u0001"+ + "\u0000\u0000\u0000\u0018\u0080\u0001\u0000\u0000\u0000\u001a\u0085\u0001"+ + "\u0000\u0000\u0000\u001c\u001e\u0003\u0002\u0001\u0000\u001d\u001c\u0001"+ + "\u0000\u0000\u0000\u001d\u001e\u0001\u0000\u0000\u0000\u001e\u001f\u0001"+ + "\u0000\u0000\u0000\u001f \u0005\u0000\u0000\u0001 \u0001\u0001\u0000\u0000"+ + "\u0000!\"\u0006\u0001\uffff\uffff\u0000\"#\u0003\u0004\u0002\u0000#)\u0001"+ + "\u0000\u0000\u0000$%\n\u0002\u0000\u0000%&\u0007\u0000\u0000\u0000&(\u0003"+ + "\u0002\u0001\u0002\'$\u0001\u0000\u0000\u0000(+\u0001\u0000\u0000\u0000"+ + ")\'\u0001\u0000\u0000\u0000)*\u0001\u0000\u0000\u0000*\u0003\u0001\u0000"+ + "\u0000\u0000+)\u0001\u0000\u0000\u0000,5\u0003\u0006\u0003\u0000-5\u0003"+ + "\b\u0004\u0000.5\u0003\f\u0006\u0000/5\u0003\n\u0005\u000005\u0003\u0012"+ + "\t\u000015\u0003\u000e\u0007\u000025\u0003\u0014\n\u000035\u0003\u0016"+ + "\u000b\u00004,\u0001\u0000\u0000\u00004-\u0001\u0000\u0000\u00004.\u0001"+ + "\u0000\u0000\u00004/\u0001\u0000\u0000\u000040\u0001\u0000\u0000\u0000"+ + "41\u0001\u0000\u0000\u000042\u0001\u0000\u0000\u000043\u0001\u0000\u0000"+ + "\u00005\u0005\u0001\u0000\u0000\u000067\u0005\u0004\u0000\u000078\u0003"+ + "\u0004\u0002\u00008\u0007\u0001\u0000\u0000\u00009:\u0003\u001a\r\u0000"+ + ":;\u0005\u0005\u0000\u0000;<\u0005\f\u0000\u0000<=\u0003\u0002\u0001\u0000"+ + "=>\u0005\r\u0000\u0000>\t\u0001\u0000\u0000\u0000?@\u0005\u0010\u0000"+ + "\u0000@B\u0005\u0005\u0000\u0000A?\u0001\u0000\u0000\u0000AB\u0001\u0000"+ + "\u0000\u0000BC\u0001\u0000\u0000\u0000CD\u0005\u0010\u0000\u0000D\u000b"+ + "\u0001\u0000\u0000\u0000EF\u0005\n\u0000\u0000FG\u0003\u0002\u0001\u0000"+ + "GH\u0005\u000b\u0000\u0000H\r\u0001\u0000\u0000\u0000IJ\u0003\u001a\r"+ + "\u0000JK\u0007\u0001\u0000\u0000KL\u0003\u0010\b\u0000L\u000f\u0001\u0000"+ + "\u0000\u0000MO\u0007\u0002\u0000\u0000NM\u0001\u0000\u0000\u0000OP\u0001"+ + "\u0000\u0000\u0000PN\u0001\u0000\u0000\u0000PQ\u0001\u0000\u0000\u0000"+ + "QT\u0001\u0000\u0000\u0000RT\u0005\u000f\u0000\u0000SN\u0001\u0000\u0000"+ + "\u0000SR\u0001\u0000\u0000\u0000T\u0011\u0001\u0000\u0000\u0000UV\u0003"+ + "\u001a\r\u0000VW\u0005\u0005\u0000\u0000WX\u0005\u0010\u0000\u0000X\u0013"+ + "\u0001\u0000\u0000\u0000YZ\u0003\u001a\r\u0000Z[\u0005\u0005\u0000\u0000"+ + "[\\\u0003\u0018\f\u0000\\d\u0001\u0000\u0000\u0000]^\u0003\u001a\r\u0000"+ + "^_\u0005\u0005\u0000\u0000_`\u0005\n\u0000\u0000`a\u0003\u0018\f\u0000"+ + "ab\u0005\u000b\u0000\u0000bd\u0001\u0000\u0000\u0000cY\u0001\u0000\u0000"+ + "\u0000c]\u0001\u0000\u0000\u0000d\u0015\u0001\u0000\u0000\u0000ek\u0003"+ + "\u0018\f\u0000fg\u0005\n\u0000\u0000gh\u0003\u0018\f\u0000hi\u0005\u000b"+ + "\u0000\u0000ik\u0001\u0000\u0000\u0000je\u0001\u0000\u0000\u0000jf\u0001"+ + "\u0000\u0000\u0000k\u0017\u0001\u0000\u0000\u0000ln\u0007\u0003\u0000"+ + "\u0000ml\u0001\u0000\u0000\u0000mn\u0001\u0000\u0000\u0000np\u0001\u0000"+ + "\u0000\u0000oq\u0007\u0002\u0000\u0000po\u0001\u0000\u0000\u0000qr\u0001"+ + "\u0000\u0000\u0000rp\u0001\u0000\u0000\u0000rs\u0001\u0000\u0000\u0000"+ + "su\u0001\u0000\u0000\u0000tv\u0007\u0003\u0000\u0000ut\u0001\u0000\u0000"+ + "\u0000uv\u0001\u0000\u0000\u0000v\u0081\u0001\u0000\u0000\u0000wy\u0007"+ + "\u0000\u0000\u0000xz\u0007\u0003\u0000\u0000yx\u0001\u0000\u0000\u0000"+ + "yz\u0001\u0000\u0000\u0000z\u0081\u0001\u0000\u0000\u0000{}\u0005\u0004"+ + "\u0000\u0000|~\u0007\u0000\u0000\u0000}|\u0001\u0000\u0000\u0000}~\u0001"+ + "\u0000\u0000\u0000~\u0081\u0001\u0000\u0000\u0000\u007f\u0081\u0005\u000f"+ + "\u0000\u0000\u0080m\u0001\u0000\u0000\u0000\u0080w\u0001\u0000\u0000\u0000"+ + "\u0080{\u0001\u0000\u0000\u0000\u0080\u007f\u0001\u0000\u0000\u0000\u0081"+ + "\u0019\u0001\u0000\u0000\u0000\u0082\u0086\u0005\u000e\u0000\u0000\u0083"+ + "\u0086\u0005\u000f\u0000\u0000\u0084\u0086\u0005\u0010\u0000\u0000\u0085"+ + "\u0082\u0001\u0000\u0000\u0000\u0085\u0083\u0001\u0000\u0000\u0000\u0085"+ + "\u0084\u0001\u0000\u0000\u0000\u0086\u001b\u0001\u0000\u0000\u0000\u000f"+ + "\u001d)4APScjmruy}\u0080\u0085"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParser.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParser.java index 1064f901cacb8..6c2d30860221a 100644 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParser.java +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParser.java @@ -14,8 +14,8 @@ import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.atn.PredictionMode; +import org.elasticsearch.core.Strings; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -25,22 +25,14 @@ public class KqlParser { private static final Logger log = LogManager.getLogger(KqlParser.class); - public QueryBuilder parseKqlQuery(String kqlQuery, SearchExecutionContext searchExecutionContext) { - if (log.isDebugEnabled()) { - log.debug("Parsing KQL query: {}", kqlQuery); - } - - return invokeParser( - kqlQuery, - new KqlParserExecutionContext(searchExecutionContext), - KqlBaseParser::topLevelQuery, - KqlAstBuilder::toQueryBuilder - ); + public QueryBuilder parseKqlQuery(String kqlQuery, KqlParsingContext kqlParserContext) { + log.trace("Parsing KQL query: {}", kqlQuery); + return invokeParser(kqlQuery, kqlParserContext, KqlBaseParser::topLevelQuery, KqlAstBuilder::toQueryBuilder); } private T invokeParser( String kqlQuery, - KqlParserExecutionContext kqlParserExecutionContext, + KqlParsingContext kqlParsingContext, Function parseFunction, BiFunction visitor ) { @@ -59,11 +51,9 @@ private T invokeParser( ParserRuleContext tree = parseFunction.apply(parser); - if (log.isTraceEnabled()) { - log.trace("Parse tree: {}", tree.toStringTree()); - } + log.trace(() -> Strings.format("Parse tree: %s", tree.toStringTree())); - return visitor.apply(new KqlAstBuilder(kqlParserExecutionContext), tree); + return visitor.apply(new KqlAstBuilder(kqlParsingContext), tree); } private static final BaseErrorListener ERROR_LISTENER = new BaseErrorListener() { diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParserExecutionContext.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParserExecutionContext.java deleted file mode 100644 index d05c70c6b933f..0000000000000 --- a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParserExecutionContext.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.kql.parser; - -import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.mapper.AbstractScriptFieldType; -import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.query.SearchExecutionContext; - -import java.time.ZoneId; -import java.util.List; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static org.elasticsearch.core.Tuple.tuple; - -class KqlParserExecutionContext extends SearchExecutionContext { - - private static final List IGNORED_METADATA_FIELDS = List.of( - "_seq_no", - "_index_mode", - "_routing", - "_ignored", - "_nested_path", - "_field_names" - ); - - private static Predicate> searchableFieldFilter = (fieldDef) -> fieldDef.v2().isSearchable(); - - private static Predicate> ignoredFieldFilter = (fieldDef) -> IGNORED_METADATA_FIELDS.contains( - fieldDef.v1() - ); - - KqlParserExecutionContext(SearchExecutionContext source) { - super(source); - } - - public Iterable> resolveFields(KqlBaseParser.FieldNameContext fieldNameContext) { - // TODO: use index settings default field. - String fieldNamePattern = fieldNameContext != null ? ParserUtils.extractText(fieldNameContext) : "*"; - - if (fieldNameContext != null && fieldNameContext.value != null && fieldNameContext.value.getType() == KqlBaseParser.QUOTED_STRING) { - return isFieldMapped(fieldNamePattern) ? List.of(tuple(fieldNamePattern, getFieldType(fieldNamePattern))) : List.of(); - } - - return getMatchingFieldNames(fieldNamePattern).stream() - .map(fieldName -> tuple(fieldName, getFieldType(fieldName))) - .filter(searchableFieldFilter.and(Predicate.not(ignoredFieldFilter))) - .collect(Collectors.toList()); - } - - public boolean isCaseSensitive() { - // TODO: implementation - return false; - } - - public ZoneId timeZone() { - return null; - } - - public static boolean isRuntimeField(MappedFieldType fieldType) { - return fieldType instanceof AbstractScriptFieldType; - } - - public static boolean isDateField(MappedFieldType fieldType) { - return fieldType.typeName().equals(DateFieldMapper.CONTENT_TYPE); - } - - public static boolean isKeywordField(MappedFieldType fieldType) { - return fieldType.typeName().equals(KeywordFieldMapper.CONTENT_TYPE); - } -} diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java new file mode 100644 index 0000000000000..5f88080fb3ed4 --- /dev/null +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/parser/KqlParsingContext.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.kql.parser; + +import org.elasticsearch.index.mapper.AbstractScriptFieldType; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.QueryRewriteContext; + +import java.time.ZoneId; +import java.util.List; +import java.util.Set; + +public class KqlParsingContext { + + private static final List IGNORED_METADATA_FIELDS = List.of( + "_seq_no", + "_index_mode", + "_routing", + "_ignored", + "_nested_path", + "_field_names" + ); + + public static Builder builder(QueryRewriteContext queryRewriteContext) { + return new Builder(queryRewriteContext); + } + + private QueryRewriteContext queryRewriteContext; + private final boolean caseInsensitive; + private final ZoneId timeZone; + private final String defaultField; + + public KqlParsingContext(QueryRewriteContext queryRewriteContext, boolean caseInsensitive, ZoneId timeZone, String defaultField) { + this.queryRewriteContext = queryRewriteContext; + this.caseInsensitive = caseInsensitive; + this.timeZone = timeZone; + this.defaultField = defaultField; + } + + public boolean caseInsensitive() { + return caseInsensitive; + } + + public ZoneId timeZone() { + return timeZone; + } + + public String defaultField() { + return defaultField; + } + + public Set resolveFieldNames(String fieldNamePattern) { + assert fieldNamePattern != null && fieldNamePattern.isEmpty() == false : "fieldNamePattern cannot be null or empty"; + return queryRewriteContext.getMatchingFieldNames(fieldNamePattern); + } + + public Set resolveDefaultFieldNames() { + return resolveFieldNames(defaultField); + } + + public MappedFieldType fieldType(String fieldName) { + return queryRewriteContext.getFieldType(fieldName); + } + + public static boolean isRuntimeField(MappedFieldType fieldType) { + return fieldType instanceof AbstractScriptFieldType; + } + + public static boolean isDateField(MappedFieldType fieldType) { + return fieldType.typeName().equals(DateFieldMapper.CONTENT_TYPE); + } + + public static boolean isKeywordField(MappedFieldType fieldType) { + return fieldType.typeName().equals(KeywordFieldMapper.CONTENT_TYPE); + } + + public static boolean isSearchableField(String fieldName, MappedFieldType fieldType) { + return IGNORED_METADATA_FIELDS.contains(fieldName) == false && fieldType.isSearchable(); + } + + public boolean isSearchableField(String fieldName) { + return isSearchableField(fieldName, fieldType(fieldName)); + } + + public static class Builder { + private final QueryRewriteContext queryRewriteContext; + private boolean caseInsensitive = true; + private ZoneId timeZone = null; + private String defaultField = null; + + private Builder(QueryRewriteContext queryRewriteContext) { + this.queryRewriteContext = queryRewriteContext; + } + + public KqlParsingContext build() { + return new KqlParsingContext(queryRewriteContext, caseInsensitive, timeZone, defaultField); + } + + public Builder caseInsensitive(boolean caseInsensitive) { + this.caseInsensitive = caseInsensitive; + return this; + } + + public Builder timeZone(ZoneId timeZone) { + this.timeZone = timeZone; + return this; + } + + public Builder defaultField(String defaultField) { + this.defaultField = defaultField; + return this; + } + } +} diff --git a/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java new file mode 100644 index 0000000000000..5dff9126b6be4 --- /dev/null +++ b/x-pack/plugin/kql/src/main/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilder.java @@ -0,0 +1,199 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.kql.query; + +import org.apache.lucene.search.Query; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.kql.parser.KqlParser; +import org.elasticsearch.xpack.kql.parser.KqlParsingContext; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.Objects; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class KqlQueryBuilder extends AbstractQueryBuilder { + public static final String NAME = "kql"; + public static final ParseField QUERY_FIELD = new ParseField("query"); + private static final ParseField CASE_INSENSITIVE_FIELD = new ParseField("case_insensitive"); + private static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); + private static final ParseField DEFAULT_FIELD_FIELD = new ParseField("default_field"); + + private static final Logger log = LogManager.getLogger(KqlQueryBuilder.class); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, a -> { + KqlQueryBuilder kqlQuery = new KqlQueryBuilder((String) a[0]); + + if (a[1] != null) { + kqlQuery.caseInsensitive((Boolean) a[1]); + } + + if (a[2] != null) { + kqlQuery.timeZone((String) a[2]); + } + + if (a[3] != null) { + kqlQuery.defaultField((String) a[3]); + } + + return kqlQuery; + }); + + static { + PARSER.declareString(constructorArg(), QUERY_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), CASE_INSENSITIVE_FIELD); + PARSER.declareString(optionalConstructorArg(), TIME_ZONE_FIELD); + PARSER.declareString(optionalConstructorArg(), DEFAULT_FIELD_FIELD); + declareStandardFields(PARSER); + } + + private final String query; + private boolean caseInsensitive = true; + private ZoneId timeZone; + private String defaultField; + + public KqlQueryBuilder(String query) { + this.query = Objects.requireNonNull(query, "query can not be null"); + } + + public KqlQueryBuilder(StreamInput in) throws IOException { + super(in); + query = in.readString(); + caseInsensitive = in.readBoolean(); + timeZone = in.readOptionalZoneId(); + defaultField = in.readOptionalString(); + } + + public static KqlQueryBuilder fromXContent(XContentParser parser) { + try { + return PARSER.apply(parser, null); + } catch (IllegalArgumentException e) { + throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e); + } + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.KQL_QUERY_ADDED; + } + + public String queryString() { + return query; + } + + public boolean caseInsensitive() { + return caseInsensitive; + } + + public KqlQueryBuilder caseInsensitive(boolean caseInsensitive) { + this.caseInsensitive = caseInsensitive; + return this; + } + + public ZoneId timeZone() { + return timeZone; + } + + public KqlQueryBuilder timeZone(String timeZone) { + this.timeZone = timeZone != null ? ZoneId.of(timeZone) : null; + return this; + } + + public String defaultField() { + return defaultField; + } + + public KqlQueryBuilder defaultField(String defaultField) { + this.defaultField = defaultField; + return this; + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + { + builder.field(QUERY_FIELD.getPreferredName(), query); + builder.field(CASE_INSENSITIVE_FIELD.getPreferredName(), caseInsensitive); + + if (defaultField != null) { + builder.field(DEFAULT_FIELD_FIELD.getPreferredName(), defaultField); + } + + if (timeZone != null) { + builder.field(TIME_ZONE_FIELD.getPreferredName(), timeZone.getId()); + } + + boostAndQueryNameToXContent(builder); + } + builder.endObject(); + } + + @Override + protected QueryBuilder doIndexMetadataRewrite(QueryRewriteContext context) throws IOException { + KqlParser parser = new KqlParser(); + QueryBuilder rewrittenQuery = parser.parseKqlQuery(query, createKqlParserContext(context)); + + log.trace(() -> Strings.format("KQL query %s translated to Query DSL: %s", query, Strings.toString(rewrittenQuery))); + + return rewrittenQuery; + } + + @Override + protected Query doToQuery(SearchExecutionContext context) throws IOException { + throw new IllegalStateException("The query should have been rewritten"); + } + + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeString(query); + out.writeBoolean(caseInsensitive); + out.writeOptionalZoneId(timeZone); + out.writeOptionalString(defaultField); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + protected int doHashCode() { + return Objects.hash(query, caseInsensitive, timeZone, defaultField); + } + + @Override + protected boolean doEquals(KqlQueryBuilder other) { + return Objects.equals(query, other.query) + && Objects.equals(timeZone, other.timeZone) + && Objects.equals(defaultField, other.defaultField) + && caseInsensitive == other.caseInsensitive; + } + + private KqlParsingContext createKqlParserContext(QueryRewriteContext queryRewriteContext) { + return KqlParsingContext.builder(queryRewriteContext) + .caseInsensitive(caseInsensitive) + .timeZone(timeZone) + .defaultField(defaultField) + .build(); + } +} diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java index 88c63e9a2585b..588e60bd4dd75 100644 --- a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java +++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java @@ -7,19 +7,22 @@ package org.elasticsearch.xpack.kql.parser; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.Predicates; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.MatchPhraseQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.test.AbstractBuilderTestCase; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.BufferedReader; import java.io.IOException; @@ -37,6 +40,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; @@ -47,6 +51,43 @@ public abstract class AbstractKqlParserTestCase extends AbstractBuilderTestCase protected static final String UNSUPPORTED_QUERY_FILE_PATH = "/unsupported-queries"; protected static final Predicate BOOLEAN_QUERY_FILTER = (q) -> q.matches("(?i)[^{]*[^\\\\]*(NOT|AND|OR)[^}]*"); + protected static final String NESTED_FIELD_NAME = "mapped_nested"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties"); + + mapping.startObject(TEXT_FIELD_NAME).field("type", "text").endObject(); + mapping.startObject(NESTED_FIELD_NAME); + { + mapping.field("type", "nested"); + mapping.startObject("properties"); + { + mapping.startObject(TEXT_FIELD_NAME).field("type", "text").endObject(); + mapping.startObject(KEYWORD_FIELD_NAME).field("type", "keyword").endObject(); + mapping.startObject(INT_FIELD_NAME).field("type", "integer").endObject(); + mapping.startObject(NESTED_FIELD_NAME); + { + mapping.field("type", "nested"); + mapping.startObject("properties"); + { + mapping.startObject(TEXT_FIELD_NAME).field("type", "text").endObject(); + mapping.startObject(KEYWORD_FIELD_NAME).field("type", "keyword").endObject(); + mapping.startObject(INT_FIELD_NAME).field("type", "integer").endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + + mapping.endObject().endObject().endObject(); + + mapperService.merge("_doc", new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE); + } + protected static String wrapWithRandomWhitespaces(String input) { return String.join("", randomWhitespaces(), input, randomWhitespaces()); } @@ -94,7 +135,18 @@ private static InputStream readFromJarUrl(URL source) throws IOException { protected List mappedLeafFields() { return Stream.concat( Arrays.stream(MAPPED_LEAF_FIELD_NAMES), - List.of(DATE_FIELD_NAME, INT_FIELD_NAME).stream().map(subfieldName -> OBJECT_FIELD_NAME + "." + subfieldName) + Stream.of( + // Adding mapped_object subfields + Strings.format("%s.%s", OBJECT_FIELD_NAME, INT_FIELD_NAME), + Strings.format("%s.%s", OBJECT_FIELD_NAME, DATE_FIELD_NAME), + // Adding mapped_nested subfields + Strings.format("%s.%s", NESTED_FIELD_NAME, TEXT_FIELD_NAME), + Strings.format("%s.%s", NESTED_FIELD_NAME, KEYWORD_FIELD_NAME), + Strings.format("%s.%s", NESTED_FIELD_NAME, INT_FIELD_NAME), + Strings.format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, TEXT_FIELD_NAME), + Strings.format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, KEYWORD_FIELD_NAME), + Strings.format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, INT_FIELD_NAME) + ) ).toList(); } @@ -111,9 +163,8 @@ protected List searchableFields(String fieldNamePattern) { protected QueryBuilder parseKqlQuery(String kqlQuery) { KqlParser parser = new KqlParser(); - SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); - - return parser.parseKqlQuery(kqlQuery, searchExecutionContext); + KqlParsingContext kqlParserContext = KqlParsingContext.builder(createQueryRewriteContext()).build(); + return parser.parseKqlQuery(kqlQuery, kqlParserContext); } protected static void assertMultiMatchQuery(QueryBuilder query, String expectedValue, MultiMatchQueryBuilder.Type expectedType) { diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java new file mode 100644 index 0000000000000..2bc23c7d457dd --- /dev/null +++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java @@ -0,0 +1,286 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.kql.query; + +import org.apache.lucene.search.Query; +import org.elasticsearch.core.Strings; +import org.elasticsearch.index.query.MultiMatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.QueryStringQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.xpack.kql.KqlPlugin; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class KqlQueryBuilderTests extends AbstractQueryTestCase { + + @Override + protected Collection> getPlugins() { + return List.of(KqlPlugin.class); + } + + @Override + protected KqlQueryBuilder doCreateTestQueryBuilder() { + KqlQueryBuilder kqlQueryBuilder = new KqlQueryBuilder(generateRandomKqlQuery()); + + if (randomBoolean()) { + kqlQueryBuilder.caseInsensitive(randomBoolean()); + } + + if (randomBoolean()) { + kqlQueryBuilder.timeZone(randomTimeZone().getID()); + } + + if (randomBoolean()) { + kqlQueryBuilder.defaultField(randomFrom("*", "mapped_*", KEYWORD_FIELD_NAME, TEXT_FIELD_NAME)); + } + + return kqlQueryBuilder; + } + + @Override + public KqlQueryBuilder mutateInstance(KqlQueryBuilder instance) throws IOException { + if (randomBoolean()) { + // Change name or boost. + return super.mutateInstance(instance); + } + + KqlQueryBuilder kqlQueryBuilder = new KqlQueryBuilder(randomValueOtherThan(instance.queryString(), this::generateRandomKqlQuery)) + .caseInsensitive(instance.caseInsensitive()) + .timeZone(instance.timeZone() != null ? instance.timeZone().getId() : null) + .defaultField(instance.defaultField()); + + if (kqlQueryBuilder.queryString().equals(instance.queryString()) == false) { + return kqlQueryBuilder; + } + + switch (randomInt() % 3) { + case 0 -> { + kqlQueryBuilder.caseInsensitive(instance.caseInsensitive() == false); + } + case 1 -> { + if (randomBoolean() && instance.defaultField() != null) { + kqlQueryBuilder.defaultField(null); + } else { + kqlQueryBuilder.defaultField( + randomValueOtherThan( + instance.defaultField(), + () -> randomFrom("*", "mapped_*", KEYWORD_FIELD_NAME, TEXT_FIELD_NAME) + ) + ); + } + } + default -> { + if (randomBoolean() && instance.timeZone() != null) { + kqlQueryBuilder.timeZone(null); + } else if (instance.timeZone() != null) { + kqlQueryBuilder.timeZone(randomValueOtherThan(instance.timeZone().getId(), () -> randomTimeZone().getID())); + } else { + kqlQueryBuilder.timeZone(randomTimeZone().getID()); + } + } + } + ; + + return kqlQueryBuilder; + } + + @Override + protected void doAssertLuceneQuery(KqlQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { + // We're not validating the query content here because it would be too complex. + // Instead, we use ad-hoc parser tests with a predictable output. + } + + private String generateRandomKqlQuery() { + return Stream.generate(() -> { + Stream terms = Stream.generate( + () -> randomValueOtherThanMany(s -> s.toLowerCase(Locale.ROOT).contains("now"), () -> randomAlphaOfLengthBetween(4, 10)) + ).limit(randomIntBetween(1, 5)); + + String subQuery = terms.collect(Collectors.joining(" ")); + + if (randomBoolean() && subQuery.isEmpty() == false) { + String operator = randomFrom(":", "<", "<=", ">", ">="); + String fieldName = randomFrom(KEYWORD_FIELD_NAME, TEXT_FIELD_NAME); + if (operator.equals(":")) { + subQuery = switch (randomFrom(0, 2)) { + case 0 -> subQuery; + case 1 -> '(' + subQuery + ')'; + default -> '"' + subQuery + '"'; + }; + } else { + fieldName = randomFrom(KEYWORD_FIELD_NAME, TEXT_FIELD_NAME, DOUBLE_FIELD_NAME, INT_FIELD_NAME); + if (List.of(DOUBLE_FIELD_NAME, INT_FIELD_NAME).contains(fieldName)) { + subQuery = String.valueOf(randomDouble()); + } + subQuery = randomBoolean() ? '"' + subQuery + '"' : subQuery; + } + + subQuery = fieldName + operator + subQuery; + } + + if (randomBoolean() && subQuery.isEmpty() == false) { + subQuery = '(' + subQuery + ')'; + } + + if (randomBoolean()) { + subQuery = "NOT " + subQuery; + } + + if (randomBoolean() && subQuery.isEmpty() == false) { + subQuery = '(' + subQuery + ')'; + } + + return subQuery; + }).limit(randomIntBetween(0, 5)).collect(Collectors.joining(randomFrom(" OR ", " AND "))); + } + + @Override + public void testMustRewrite() throws IOException { + SearchExecutionContext context = createSearchExecutionContext(); + context.setAllowUnmappedFields(true); + KqlQueryBuilder queryBuilder = createTestQueryBuilder(); + IllegalStateException e = assertThrows(IllegalStateException.class, () -> queryBuilder.toQuery(context)); + assertThat(e.getMessage(), Matchers.containsString("The query should have been rewritten")); + } + + public void testCaseInsensitiveWildcardQuery() throws IOException { + QueryRewriteContext queryRewriteContext = createQueryRewriteContext(); + SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); + + for (boolean caseInsensitive : List.of(true, false)) { + KqlQueryBuilder kqlQuery = new KqlQueryBuilder(KEYWORD_FIELD_NAME + ": foo*"); + // Check case case_insensitive is true by default + assertThat(kqlQuery.caseInsensitive(), equalTo(true)); + + kqlQuery.caseInsensitive(caseInsensitive); + + ; + assertThat( + asInstanceOf(WildcardQueryBuilder.class, rewriteQuery(kqlQuery, queryRewriteContext, searchExecutionContext)) + .caseInsensitive(), + equalTo(caseInsensitive) + ); + } + } + + public void testCaseInsensitiveTermQuery() throws IOException { + QueryRewriteContext queryRewriteContext = createQueryRewriteContext(); + SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); + + for (boolean caseInsensitive : List.of(true, false)) { + KqlQueryBuilder kqlQuery = new KqlQueryBuilder(KEYWORD_FIELD_NAME + ": foo"); + // Check case case_insensitive is true by default + assertThat(kqlQuery.caseInsensitive(), equalTo(true)); + + kqlQuery.caseInsensitive(caseInsensitive); + + assertThat( + asInstanceOf(TermQueryBuilder.class, rewriteQuery(kqlQuery, queryRewriteContext, searchExecutionContext)).caseInsensitive(), + equalTo(caseInsensitive) + ); + } + } + + public void testTimeZone() throws IOException { + QueryRewriteContext queryRewriteContext = createQueryRewriteContext(); + SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); + String timeZone = randomTimeZone().getID(); + + for (String operator : List.of(":", "<", "<=", ">", ">=")) { + KqlQueryBuilder kqlQuery = new KqlQueryBuilder(Strings.format("%s %s %s", DATE_FIELD_NAME, operator, "2018-03-28")); + assertThat(kqlQuery.timeZone(), nullValue()); // timeZone is not set by default. + kqlQuery.timeZone(timeZone); + + assertThat( + asInstanceOf(RangeQueryBuilder.class, rewriteQuery(kqlQuery, queryRewriteContext, searchExecutionContext)).timeZone(), + equalTo(timeZone) + ); + } + } + + public void testDefaultFieldWildcardQuery() throws IOException { + QueryRewriteContext queryRewriteContext = createQueryRewriteContext(); + SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); + KqlQueryBuilder kqlQuery = new KqlQueryBuilder(Strings.format("foo*")); + assertThat(kqlQuery.defaultField(), nullValue()); // default_field is not set by default. + + kqlQuery.defaultField(TEXT_FIELD_NAME); + + assertThat( + asInstanceOf(QueryStringQueryBuilder.class, rewriteQuery(kqlQuery, queryRewriteContext, searchExecutionContext)).defaultField(), + equalTo(TEXT_FIELD_NAME) + ); + } + + public void testDefaultFieldMatchQuery() throws IOException { + + QueryRewriteContext queryRewriteContext = createQueryRewriteContext(); + SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); + + { + // Using a specific field name + KqlQueryBuilder kqlQuery = new KqlQueryBuilder(Strings.format("foo")); + assertThat(kqlQuery.defaultField(), nullValue()); // default_field is not set by default. + + kqlQuery.defaultField(TEXT_FIELD_NAME); + MultiMatchQueryBuilder rewritenQuery = asInstanceOf( + MultiMatchQueryBuilder.class, + rewriteQuery(kqlQuery, queryRewriteContext, searchExecutionContext) + ); + assertThat(rewritenQuery.fields().keySet(), contains(TEXT_FIELD_NAME)); + } + + { + // Using a pattern for as the field name + KqlQueryBuilder kqlQuery = new KqlQueryBuilder(Strings.format("foo")); + assertThat(kqlQuery.defaultField(), nullValue()); // default_field is not set by default. + + kqlQuery.defaultField("mapped_object.*"); + MultiMatchQueryBuilder rewritenQuery = asInstanceOf( + MultiMatchQueryBuilder.class, + rewriteQuery(kqlQuery, queryRewriteContext, searchExecutionContext) + ); + assertThat(rewritenQuery.fields().keySet(), contains("mapped_object.mapped_date", "mapped_object.mapped_int")); + } + } + + public void testQueryNameIsPreserved() throws IOException { + QueryRewriteContext queryRewriteContext = createQueryRewriteContext(); + SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); + + KqlQueryBuilder kqlQuery = new KqlQueryBuilder(generateRandomKqlQuery()).queryName(randomIdentifier()); + QueryBuilder rewrittenQuery = rewriteQuery(kqlQuery, queryRewriteContext, searchExecutionContext); + assertThat(rewrittenQuery.queryName(), equalTo(kqlQuery.queryName())); + } + + public void testQueryBoostIsPreserved() throws IOException { + QueryRewriteContext queryRewriteContext = createQueryRewriteContext(); + SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); + + KqlQueryBuilder kqlQuery = new KqlQueryBuilder(generateRandomKqlQuery()).boost(randomFloatBetween(0, Float.MAX_VALUE, true)); + QueryBuilder rewrittenQuery = rewriteQuery(kqlQuery, queryRewriteContext, searchExecutionContext); + assertThat(rewrittenQuery.boost(), equalTo(kqlQuery.boost())); + } +} diff --git a/x-pack/plugin/kql/src/test/resources/supported-queries b/x-pack/plugin/kql/src/test/resources/supported-queries index 4911c9e3ebecd..b659b1ae5b1db 100644 --- a/x-pack/plugin/kql/src/test/resources/supported-queries +++ b/x-pack/plugin/kql/src/test/resources/supported-queries @@ -23,54 +23,54 @@ f*oo *:"foo bar" // Querying a field -foo_field:200 -foo_field:foo -foo_field:foo bar -foo_field:(foo bar) -foo_field:foo* -foo_field: f*oo -foo_field: *foo -foo_field:"foo bar" -foo_field.subfield:foo -foo_*_field:foo -foo_field:* -foo_*:* +mapped_int:200 +mapped_string_2:foo +mapped_string:foo bar +mapped_string:(foo bar) +mapped_string:foo* +mapped_string_2: f*oo +mapped_string: *foo +mapped_string:"foo bar" +mapped_object.subfield:foo +mapped_str*:foo +mapped_string:* +mapped_str_*:* // Range queries -foo_field<200 -foo_field=200 -foo_field>=foo -foo_field>"foo bar" -foo_field<=foo -foo_field>=foo +mapped_int<200 +mapped_string_2=200 +mapped_string_alias>=foo +mapped_string>"foo bar" +mapped_string<=foo +mapped_string_2>=foo // Boolean queries NOT foo NOT foo bar -NOT foo_field:foo -NOT foo_fieldbar -(foo_field:foo) AND (foo_field:foo bar) -foo_field:foo OR foo_field:foo bar -NOT(foo_field:foo OR foo_field:foo bar) -NOT(foo_field:foo AND foo_field:foo bar) -NOT foo_field:foo AND NOT foo_field:foo bar -(NOT foo_field:foo) AND (NOT foo_field:foo bar) -NOT(foo_field:foo) AND NOT(foo_field:foo bar) -foo_field:foo AND foo_field:foo bar AND foo bar -foo_field:foo AND foo_field:foo bar OR foo bar -foo_field:foo OR foo_field:foo bar OR foo bar -foo_field:foo OR foo_field:foo bar AND foo bar -foo_field:foo AND (foo_field:foo bar OR foo bar) -foo_field:foo AND (foo_field:foo bar OR foo bar) -foo_field:foo OR (foo_field:foo bar OR foo bar) +NOT mapped_string:foo +NOT mapped_string_2bar +(mapped_string:foo) AND (mapped_string:foo bar) +mapped_string:foo OR mapped_string_2:foo bar +NOT(mapped_string:foo OR mapped_string:foo bar) +NOT(mapped_string:foo AND mapped_string:foo bar) +NOT mapped_string:foo AND NOT mapped_string_2:foo bar +(NOT mapped_string_alias:foo) AND (NOT mapped_string:foo bar) +NOT(mapped_string:foo) AND NOT(mapped_string:foo bar) +mapped_string:foo AND mapped_string_2:foo bar AND foo bar +mapped_string:foo AND mapped_string_2:foo bar OR foo bar +mapped_string:foo OR mapped_string_2:foo bar OR foo bar +mapped_string:foo OR mapped_string:foo bar AND foo bar +mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) +mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) +mapped_string:foo OR (mapped_string_2:foo bar OR foo bar) -foo:AND -foo:OR -foo:NOT +mapped_string:AND +mapped_string:OR +mapped_string:NOT foo AND foo OR foo NOT @@ -79,43 +79,51 @@ OR foo NOT // Nested queries -nested_field: { NOT foo } -nested_field: { NOT foo bar } -nested_field: { NOT foo_field:foo } -nested_field: { foo_field:foo AND foo_field:foo bar } -nested_field: { foo_fieldbar } -nested_field: { (foo_field:foo) AND (foo_field:foo bar) } -nested_field: { foo_field:foo OR foo_field:foo bar } -nested_field: { NOT(foo_field:foo OR foo_field:foo bar) } -nested_field: { NOT(foo_field:foo AND foo_field:foo bar) } -nested_field: { NOT foo_field:foo AND NOT foo_field:foo bar } -nested_field: { (NOT foo_field:foo) AND (NOT foo_field:foo bar) } -nested_field: { NOT(foo_field:foo) AND NOT(foo_field:foo bar) } -nested_field: { foo_field:foo AND foo_field:foo bar AND foo bar } -nested_field: { foo_field:foo AND foo_field:foo bar OR foo bar } -nested_field: { foo_field:foo OR foo_field:foo bar OR foo bar } -nested_field: { foo_field:foo OR foo_field:foo bar AND foo bar } -nested_field: { foo_field:foo AND (foo_field:foo bar OR foo bar) } -nested_field: { foo_field:foo AND (foo_field:foo bar OR foo bar) } -nested_field: { foo_field:foo OR (foo_field:foo bar OR foo bar) } -nested_field: { sub_nested_field : { foo_field:foo } AND foo_field:foo bar } +mapped_nested: { NOT foo } +mapped_nested: { NOT foo bar } +mapped_nested: { NOT mapped_string:foo } +mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar } +mapped_nested: { mapped_string2 } +mapped_nested: { (mapped_string:foo) AND (mapped_string_2:foo bar) } +mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar } +mapped_nested: { NOT(mapped_string:foo OR mapped_string_2:foo bar) } +mapped_nested: { NOT(mapped_string:foo AND mapped_string_2:foo bar) } +mapped_nested: { NOT mapped_string:foo AND NOT mapped_string_2:foo bar } +mapped_nested: { (NOT mapped_string:foo) AND (NOT mapped_string_2:foo bar) } +mapped_nested: { NOT(mapped_string:foo) AND NOT(mapped_string_2:foo bar) } +mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar AND foo bar } +mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar OR foo bar } +mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar OR foo bar } +mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar AND foo bar } +mapped_nested: { mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) } +mapped_nested: { mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) } +mapped_nested: { mapped_string:foo OR (mapped_string_2:foo bar OR foo bar) } +mapped_nested: { mapped_str*:foo } +mapped_nested: { mapped_nested : { mapped_string:foo AND mapped_int < 3 } AND mapped_string_2:foo bar } +mapped_nested: { mapped_nested.mapped_string:foo AND mapped_string_2:foo bar } + +// Inline nested queries +mapped_nested.mapped_string:foo AND mapped_nested.mapped_int < 2 +mapped_nested.mapped_nested.mapped_string:foo AND mapped_nested.mapped_int < 2 +mapped_nested.mapped_str*: foo + // Queries with escape sequences -foo_field : (foo\(bar\)) -foo_field : foo\:bar -foo_field : (foo \\and bar) -foo_field : (foo \\or bar) -foo_field : foo \\not bar -foo_field : foo \{bar\} -foo_field : foo \(bar\) -foo_field : foo \\ bar -foo_field : foo \"bar\" +mapped_string:(foo\(bar\)) +mapped_string:foo\:bar +mapped_string:(foo \\and bar) +mapped_string:(foo \\or bar) +mapped_string:foo \\not bar +mapped_string:foo \{bar\} +mapped_string:foo \(bar\) +mapped_string:foo \\ bar +mapped_string:foo \"bar\" -foo_field : "foo and bar" -foo_field : "foo not bar" -foo_field : "foo or bar" -foo_field : "foo : bar" -foo_field : "foo { bar }" -foo_field : "foo (bar)" -foo_field : "foo \\ bar" -foo_field : "foo \"bar\"" +mapped_string:"foo and bar" +mapped_string:"foo not bar" +mapped_string:"foo or bar" +mapped_string:"foo : bar" +mapped_string:"foo { bar }" +mapped_string:"foo (bar)" +mapped_string:"foo \\ bar" +mapped_string:"foo \"bar\"" diff --git a/x-pack/plugin/kql/src/test/resources/unsupported-queries b/x-pack/plugin/kql/src/test/resources/unsupported-queries index 64901891c6786..149bcf5bd2b5a 100644 --- a/x-pack/plugin/kql/src/test/resources/unsupported-queries +++ b/x-pack/plugin/kql/src/test/resources/unsupported-queries @@ -1,36 +1,36 @@ // Incomplete expressions -foo_field : -foo_field < -foo_field > -foo_field >= -foo_field <= +mapped_string : +mapped_string < +mapped_string > +mapped_string >= +mapped_string <= >= foo : "foo" : foo // Parentheses mismatch -foo_field: (foo bar -foo_field: foo bar) -NOT foo_field:foo OR foo_field:foo bar) -NOT (foo_field:foo AND) foo_field:foo bar +mapped_string: (foo bar +mapped_string: foo bar) +NOT mapped_string:foo OR mapped_string_2:foo bar) +NOT (mapped_string:foo AND) mapped_string_2:foo bar // Quotes mismatch -foo_field: "foo bar -foo_field: foo bar" +mapped_string: "foo bar +mapped_string: foo bar" // Can't nest grouping terms parentheses -foo_field:(foo (bar)) +mapped_string:(foo (bar)) // Bad syntax for nested fields: -nested_field { foo: bar } +mapped_nested { mapped_string: bar } // Missing escape sequences: -foo_field: foo:bar -foo_field: (foo and bar) -foo_field: (foo or bar) -foo_field: foo not bar -foo_field: foo { bar } -foo_field: foo (bar) -foo_field: foo "bar" -foo_field: "foo "bar"" +mapped_string: foo:bar +mapped_string: (foo and bar) +mapped_string: (foo or bar) +mapped_string: foo not bar +mapped_string: foo { bar } +mapped_string: foo (bar) +mapped_string: foo "bar" +mapped_string: "foo "bar"" diff --git a/x-pack/plugin/kql/src/yamlRestTest/java/org/elasticsearch/xpack/kql/KqlRestIT.java b/x-pack/plugin/kql/src/yamlRestTest/java/org/elasticsearch/xpack/kql/KqlRestIT.java new file mode 100644 index 0000000000000..35df46b0fdcbb --- /dev/null +++ b/x-pack/plugin/kql/src/yamlRestTest/java/org/elasticsearch/xpack/kql/KqlRestIT.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.kql; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +public class KqlRestIT extends ESClientYamlSuiteTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .setting("xpack.security.enabled", "false") + .setting("xpack.security.http.ssl.enabled", "false") + .distribution(DistributionType.DEFAULT) + .build(); + + public KqlRestIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/10_kql_basic_query.yml b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/10_kql_basic_query.yml new file mode 100644 index 0000000000000..bb59c6a48b612 --- /dev/null +++ b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/10_kql_basic_query.yml @@ -0,0 +1,212 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ kql_query ] + test_runner_features: capabilities + reason: KQL query is not available + + - do: + indices.create: + index: test-index + body: + mappings: + properties: + date_field: + type: date + text_field: + type: text + keyword_field: + type: keyword + integer_field: + type: integer + double_field: + type: double + + - do: + bulk: + index: test-index + refresh: true + body: | + { "index" : { "_id": "doc-1" } } + { "text_field": "foo bar", "integer_field": 1, "double_field": 3.5, "date_field": "2010-03-06T14:15:00", "keyword_field": "foo bar" } + { "index" : { "_id": "doc-42" } } + { "text_field": "foo baz", "integer_field": 2, "double_field": 18.9, "date_field": "2018-03-28T20:30:00", "keyword_field": "foo baz" } + +--- +"KQL match all queries": + # KQL empty query are supposed to match all. + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "" } } + } + - match: { hits.total: 2 } + + # Using the *:* syntax + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "*" } } + } + - match: { hits.total: 2 } + + # Using the *:* syntax + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "*:*" } } + } + - match: { hits.total: 2 } + +--- +"KQL match term queries (no field specified)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "bar" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "foo bar" } } + } + - match: { hits.total: 2 } + - match: { hits.hits.0._id: "doc-1" } + + # KQL does not match on the _id field when no field is specified. + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "doc-42" } } + } + - match: { hits.total: 0 } + +--- +"KQL match multiple terms queries (no field specified)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "foo bar" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "(foo bar)" } } + } + - match: { hits.total: 2 } + +--- +"KQL match phrase queries (no field specified)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "\"foo bar\"" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + +--- +"KQL match number queries (no field specified)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "2" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "3.5" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + +--- +"KQL match multiple terms queries (no matches)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "no match" } } + } + - match: { hits.total: 0 } + + +--- +"KQL boolean queries": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field: foo AND integer_field > 1" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field: baz OR keyword_field: foo bar" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "NOT text_field: baz" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + diff --git a/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml new file mode 100644 index 0000000000000..2e40c73ecf829 --- /dev/null +++ b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/20_kql_match_query.yml @@ -0,0 +1,266 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ kql_query ] + test_runner_features: capabilities + reason: KQL query is not available + + - requires: + "test_runner_features": "contains" + + - do: + indices.create: + index: test-index + body: + mappings: + properties: + date_field: + type: date + text_field: + type: text + keyword_field: + type: keyword + integer_field: + type: integer + double_field: + type: double + + - do: + bulk: + index: test-index + refresh: true + body: | + { "index" : { "_id": "doc-1" } } + { "text_field": "foo bar", "integer_field": 1, "double_field": 3.5, "date_field": "2010-03-06T14:15:00", "keyword_field": "foo bar" } + { "index" : { "_id": "doc-42" } } + { "text_field": "foo baz", "integer_field": 2, "double_field": 18.9, "date_field": "2018-03-28T20:30:00", "keyword_field": "foo baz" } + + +--- +"KQL match term queries (text field)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field:bar" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field: foo bar" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field: (foo bar)" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field: \"foo bar\"" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field: bar*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + +--- +"KQL match term queries (integer field)": + - do: + catch: bad_request + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "integer_field: foo" } } + } + - match: { error.type: "search_phase_execution_exception" } + - match: { error.root_cause.0.type: "query_shard_exception" } + - match: { error.root_cause.0.reason: "failed to create query: For input string: \"foo\"" } + - contains: { error.root_cause.0.stack_trace: "Caused by: java.lang.NumberFormatException: For input string: \"foo\"" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "integer_field: 2" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "integer_field: \"2\"" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + +--- +"KQL match term queries (double field)": + - do: + catch: bad_request + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "double_field: foo" } } + } + - match: { error.type: "search_phase_execution_exception" } + - match: { error.root_cause.0.type: "query_shard_exception" } + - match: { error.root_cause.0.reason: "failed to create query: For input string: \"foo\"" } + - contains: { error.root_cause.0.stack_trace: "Caused by: java.lang.NumberFormatException: For input string: \"foo\"" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "double_field: 18.9" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "double_field: \"18.9\"" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + +--- +"KQL match term queries (keyword field)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "keyword_field:foo bar" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "keyword_field: \"foo bar\"" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "keyword_field: foo ba*" } } + } + - match: { hits.total: 2 } + + +--- +"KQL match term queries (date field)": + - do: + catch: bad_request + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field: foo" } } + } + - match: { error.type: "search_phase_execution_exception" } + - match: { error.root_cause.0.type: "parse_exception" } + - contains: { error.root_cause.0.reason: "failed to parse date field [foo]" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field: 2010-03-06" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field: now" } } + } + - match: { hits.total: 0 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field: now/1d" } } + } + - match: { hits.total: 0 } + +--- +"KQL match term queries (search by id)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "_id:doc-1" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } diff --git a/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/30_kql_range_query.yml b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/30_kql_range_query.yml new file mode 100644 index 0000000000000..e03fd41306ba9 --- /dev/null +++ b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/30_kql_range_query.yml @@ -0,0 +1,343 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ kql_query ] + test_runner_features: capabilities + reason: KQL query is not available + + - requires: + "test_runner_features": "contains" + + - do: + indices.create: + index: test-index + body: + mappings: + properties: + date_field: + type: date + text_field: + type: text + keyword_field: + type: keyword + integer_field: + type: integer + double_field: + type: double + + - do: + bulk: + index: test-index + refresh: true + body: | + { "index" : { "_id": "doc-1" } } + { "text_field": "bar", "integer_field": 1, "double_field": 3.5, "date_field": "2010-03-06T14:15:00", "keyword_field": "foo bar" } + { "index" : { "_id": "doc-42" } } + { "text_field": "baz", "integer_field": 2, "double_field": 18.9, "date_field": "2018-03-28T20:30:00", "keyword_field": "foo baz" } + + +--- +"KQL match term queries (text field)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field < baz" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field <= baz" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field > bar" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field >= bar" } } + } + - match: { hits.total: 2 } + + +--- +"KQL match term queries (integer field)": + - do: + catch: bad_request + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "integer_field < foo" } } + } + - match: { error.type: "search_phase_execution_exception" } + - match: { error.root_cause.0.type: "query_shard_exception" } + - match: { error.root_cause.0.reason: "failed to create query: For input string: \"foo\"" } + - contains: { error.root_cause.0.stack_trace: "Caused by: java.lang.NumberFormatException: For input string: \"foo\"" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "integer_field >= 1" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "integer_field > 1" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "integer_field <= 2" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "integer_field < 2" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + +--- +"KQL match term queries (double field)": + - do: + catch: bad_request + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "double_field < foo" } } + } + - match: { error.type: "search_phase_execution_exception" } + - match: { error.root_cause.0.type: "query_shard_exception" } + - match: { error.root_cause.0.reason: "failed to create query: For input string: \"foo\"" } + - contains: { error.root_cause.0.stack_trace: "Caused by: java.lang.NumberFormatException: For input string: \"foo\"" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "double_field >= 3.5" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "double_field > 3.5" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "double_field <= 18.9" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "double_field < 18.9" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + +--- +"KQL match term queries (keyword field)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "keyword_field < foo baz" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "keyword_field <= foo baz" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "keyword_field > foo bar" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "keyword_field >= foo bar" } } + } + - match: { hits.total: 2 } + + +--- +"KQL match term queries (date field)": + - do: + catch: bad_request + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field: foo" } } + } + - match: { error.type: "search_phase_execution_exception" } + - match: { error.root_cause.0.type: "parse_exception" } + - contains: { error.root_cause.0.reason: "failed to parse date field [foo]" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field < 2018-03-28" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field <= 2018-03-28" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field > 2010-03-06" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-42" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field >= 2010-03-06" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field < now" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field <= now" } } + } + - match: { hits.total: 2 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field > now" } } + } + - match: { hits.total: 0 } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field >= now" } } + } + - match: { hits.total: 0 } diff --git a/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/40_kql_exist_query.yml b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/40_kql_exist_query.yml new file mode 100644 index 0000000000000..ca9197d382f64 --- /dev/null +++ b/x-pack/plugin/kql/src/yamlRestTest/resources/rest-api-spec/test/kql/40_kql_exist_query.yml @@ -0,0 +1,182 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ kql_query ] + test_runner_features: capabilities + reason: KQL query is not available + + - do: + indices.create: + index: test-index + body: + mappings: + properties: + date_field: + type: date + text_field: + type: text + keyword_field: + type: keyword + integer_field: + type: integer + double_field: + type: double + + - do: + bulk: + index: test-index + refresh: true + body: | + { "index" : { "_id": "doc-1" } } + { "text_field": "foo bar", "integer_field": 1, "double_field": 3.5, "date_field": "2010-03-06T14:15:00", "keyword_field": "foo bar" } + { "index" : { "_id": "doc-42" } } + { "another_field": "foo"} + +--- +"KQL exists queries - Existing field": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "integer_field:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "double_field:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "keyword_field:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_*:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "*_field:*" } } + } + - match: { hits.total: 2 } + +--- +"KQL exists queries (existing field)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "text_field:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "integer_field:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "double_field:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "date_field:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "keyword_field:*" } } + } + - match: { hits.total: 1 } + - match: { hits.hits.0._id: "doc-1" } + +--- +"KQL exists queries (non-existing field)": + - do: + search: + index: test-index + rest_total_hits_as_int: true + body: > + { + "query": { "kql": { "query": "non_existing_field:*" } } + } + - match: { hits.total: 0 } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java index 1be31df9e693d..194a9379b4b63 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java @@ -1033,6 +1033,9 @@ public void testCanMatchSkipsPartiallyMountedIndicesWhenFrozenNodesUnavailable() TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("_tier", "data_content"); List indicesToSearch = List.of(regularIndex, partiallyMountedIndex); SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0])) + // we randomise the partial search results because if shards that do NOT match the query are unavailable + // the search is not partial + .allowPartialSearchResults(randomBoolean()) .source(new SearchSourceBuilder().query(termQueryBuilder)); assertResponse(client().search(request), searchResponse -> { @@ -1049,6 +1052,7 @@ public void testCanMatchSkipsPartiallyMountedIndicesWhenFrozenNodesUnavailable() TermsQueryBuilder termsQueryBuilder = QueryBuilders.termsQuery("_tier", "data_hot", "data_content"); List indicesToSearch = List.of(regularIndex, partiallyMountedIndex); SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0])) + .allowPartialSearchResults(randomBoolean()) .source(new SearchSourceBuilder().query(termsQueryBuilder)); assertResponse(client().search(request), searchResponse -> { @@ -1065,6 +1069,7 @@ public void testCanMatchSkipsPartiallyMountedIndicesWhenFrozenNodesUnavailable() BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("_tier", "data_frozen")); List indicesToSearch = List.of(regularIndex, partiallyMountedIndex); SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0])) + .allowPartialSearchResults(randomBoolean()) .source(new SearchSourceBuilder().query(boolQueryBuilder)); assertResponse(client().search(request), searchResponse -> { @@ -1082,6 +1087,7 @@ public void testCanMatchSkipsPartiallyMountedIndicesWhenFrozenNodesUnavailable() .mustNot(randomFrom(QueryBuilders.wildcardQuery("_tier", "dat*ozen"), QueryBuilders.prefixQuery("_tier", "data_fro"))); List indicesToSearch = List.of(regularIndex, partiallyMountedIndex); SearchRequest request = new SearchRequest().indices(indicesToSearch.toArray(new String[0])) + .allowPartialSearchResults(randomBoolean()) .source(new SearchSourceBuilder().query(boolQueryBuilder)); assertResponse(client().search(request), searchResponse -> { diff --git a/x-pack/plugin/security/qa/multi-cluster/build.gradle b/x-pack/plugin/security/qa/multi-cluster/build.gradle index c7b8f81bb7876..34976163e6b2f 100644 --- a/x-pack/plugin/security/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/security/qa/multi-cluster/build.gradle @@ -31,13 +31,15 @@ dependencies { tasks.named("javaRestTest") { enabled = true // This is tested explicitly in bwc test tasks. - exclude '**/RemoteClusterSecurityBwcRestIT.class' + exclude '**/RemoteClusterSecurityBWCToRCS1ClusterRestIT.class' + exclude '**/RemoteClusterSecurityBWCToRCS2ClusterRestIT.class' } -BuildParams.bwcVersions.withWireCompatible(v -> v.before(BuildParams.isSnapshotBuild() ? '8.8.0' : '8.9.1')) { bwcVersion, baseName -> +BuildParams.bwcVersions.withWireCompatible(v -> v.onOrAfter('8.16.0')) { bwcVersion, baseName -> tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) - include '**/RemoteClusterSecurityBwcRestIT.class' + include '**/RemoteClusterSecurityBWCToRCS1ClusterRestIT.class' + include '**/RemoteClusterSecurityBWCToRCS2ClusterRestIT.class' } } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityBWCRestIT.java similarity index 65% rename from x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java rename to x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityBWCRestIT.java index 17acd258ed34b..20cdbb9f8b0df 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBwcRestIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/AbstractRemoteClusterSecurityBWCRestIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.remotecluster; +import org.apache.http.util.EntityUtils; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; @@ -15,14 +16,9 @@ import org.elasticsearch.core.Strings; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchResponseUtils; -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.elasticsearch.test.cluster.util.Version; -import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.rest.ObjectPath; -import org.junit.ClassRule; -import org.junit.rules.RuleChain; -import org.junit.rules.TestRule; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; import java.util.Arrays; @@ -32,48 +28,21 @@ import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; /** - * BWC test which ensures that users and API keys with defined {@code remote_indices} privileges can be used to query legacy remote clusters + * A set of BWC tests that can be executed with either RCS 1 or RCS 2 against an older fulfilling cluster. */ -public class RemoteClusterSecurityBwcRestIT extends AbstractRemoteClusterSecurityTestCase { +public abstract class AbstractRemoteClusterSecurityBWCRestIT extends AbstractRemoteClusterSecurityTestCase { - private static final Version OLD_CLUSTER_VERSION = Version.fromString(System.getProperty("tests.old_cluster_version")); + protected abstract boolean isRCS2(); - static { - fulfillingCluster = ElasticsearchCluster.local() - .version(OLD_CLUSTER_VERSION) - .distribution(DistributionType.DEFAULT) - .name("fulfilling-cluster") - .apply(commonClusterConfig) - .setting("xpack.ml.enabled", "false") - .build(); - - queryCluster = ElasticsearchCluster.local() - .version(Version.CURRENT) - .distribution(DistributionType.INTEG_TEST) - .name("query-cluster") - .apply(commonClusterConfig) - .setting("xpack.security.remote_cluster_client.ssl.enabled", "true") - .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") - .rolesFile(Resource.fromClasspath("roles.yml")) - .build(); - } - - @ClassRule - // Use a RuleChain to ensure that fulfilling cluster is started before query cluster - public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster); - - public void testBwcWithLegacyCrossClusterSearch() throws Exception { - final boolean useProxyMode = randomBoolean(); - // Update remote cluster settings on QC. - setupQueryClusterRemoteClusters(useProxyMode); - // Ensure remote cluster is connected - ensureRemoteFulfillingClusterIsConnected(useProxyMode); + public void testBwcCCSViaRCS1orRCS2() throws Exception { // Fulfilling cluster { @@ -122,19 +91,22 @@ public void testBwcWithLegacyCrossClusterSearch() throws Exception { ] }"""); assertOK(adminClient().performRequest(putRoleRequest)); - // We need to define the same role on QC and FC in order for CCS to work. - final var putRoleRequestFulfilling = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE); - putRoleRequestFulfilling.setJsonEntity(""" - { - "cluster": ["manage_own_api_key"], - "indices": [ + if (isRCS2() == false) { + // We need to define the same role on QC and FC in order for CCS to work. + final var putRoleRequestFulfilling = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE); + putRoleRequestFulfilling.setJsonEntity(""" { - "names": ["remote_index1"], - "privileges": ["read", "read_cross_cluster"] - } - ] - }"""); - assertOK(performRequestAgainstFulfillingCluster(putRoleRequestFulfilling)); + "cluster": ["manage_own_api_key"], + "indices": [ + { + "names": ["remote_index1"], + "privileges": ["read", "read_cross_cluster"] + } + ] + }"""); + assertOK(performRequestAgainstFulfillingCluster(putRoleRequestFulfilling)); + } + final var putUserRequest = new Request("PUT", "/_security/user/" + REMOTE_SEARCH_USER); putUserRequest.setJsonEntity(""" { @@ -166,7 +138,7 @@ public void testBwcWithLegacyCrossClusterSearch() throws Exception { ], "remote_cluster": [ { - "privileges": ["monitor_enrich"], + "privileges": ["monitor_enrich", "monitor_stats"], "clusters": ["*"] } ] @@ -187,38 +159,35 @@ public void testBwcWithLegacyCrossClusterSearch() throws Exception { // Check that we can search the fulfilling cluster from the querying cluster final boolean alsoSearchLocally = randomBoolean(); + final String remoteClusterName = randomFrom("my_remote_cluster", "*", "my_remote_*"); + final String remoteIndexName = randomFrom("remote_index1", "*"); final var searchRequest = new Request( "GET", String.format( Locale.ROOT, "/%s%s:%s/_search?ccs_minimize_roundtrips=%s", alsoSearchLocally ? "local_index," : "", - randomFrom("my_remote_cluster", "*", "my_remote_*"), - randomFrom("remote_index1", "*"), + remoteClusterName, + remoteIndexName, randomBoolean() ) ); - final String sendRequestWith = randomFrom("user", "apikey"); - final Response response = sendRequestWith.equals("user") - ? performRequestWithRemoteAccessUser(searchRequest) - : performRequestWithApiKey(searchRequest, apiKeyEncoded); + String esqlCommand = String.format(Locale.ROOT, "FROM %s,%s:%s | LIMIT 10", "local_index", remoteClusterName, remoteIndexName); + // send request with user + Response response = performRequestWithRemoteAccessUser(searchRequest); assertOK(response); - final SearchResponse searchResponse; try (var parser = responseAsParser(response)) { - searchResponse = SearchResponseUtils.parseSearchResponse(parser); + assertSearchResponse(SearchResponseUtils.parseSearchResponse(parser), alsoSearchLocally); } - try { - final List actualIndices = Arrays.stream(searchResponse.getHits().getHits()) - .map(SearchHit::getIndex) - .collect(Collectors.toList()); - if (alsoSearchLocally) { - assertThat(actualIndices, containsInAnyOrder("remote_index1", "local_index")); - } else { - assertThat(actualIndices, containsInAnyOrder("remote_index1")); - } - } finally { - searchResponse.decRef(); + assertEsqlResponse(performRequestWithRemoteAccessUser(esqlRequest(esqlCommand))); + + // send request with apikey + response = performRequestWithApiKey(searchRequest, apiKeyEncoded); + assertOK(response); + try (var parser = responseAsParser(response)) { + assertSearchResponse(SearchResponseUtils.parseSearchResponse(parser), alsoSearchLocally); } + assertEsqlResponse(performRequestWithApiKey(esqlRequest(esqlCommand), apiKeyEncoded)); } } @@ -231,6 +200,14 @@ private void ensureRemoteFulfillingClusterIsConnected(boolean useProxyMode) thro final Map remoteInfoMap = responseAsMap(remoteInfoResponse); assertThat(remoteInfoMap, hasKey("my_remote_cluster")); assertThat(org.elasticsearch.xcontent.ObjectPath.eval("my_remote_cluster.connected", remoteInfoMap), is(true)); + if (isRCS2()) { + assertThat( + org.elasticsearch.xcontent.ObjectPath.eval("my_remote_cluster.cluster_credentials", remoteInfoMap), + is("::es_redacted::") // RCS 2.0 + ); + } else { + assertThat(org.elasticsearch.xcontent.ObjectPath.eval("my_remote_cluster.cluster_credentials", remoteInfoMap), nullValue()); + } if (false == useProxyMode) { assertThat( org.elasticsearch.xcontent.ObjectPath.eval("my_remote_cluster.num_nodes_connected", remoteInfoMap), @@ -240,7 +217,17 @@ private void ensureRemoteFulfillingClusterIsConnected(boolean useProxyMode) thro }); } - private void setupQueryClusterRemoteClusters(boolean useProxyMode) throws IOException { + private Response performRequestWithRemoteAccessUser(final Request request) throws IOException { + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue(REMOTE_SEARCH_USER, PASS))); + return client().performRequest(request); + } + + private Response performRequestWithApiKey(final Request request, final String encoded) throws IOException { + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", "ApiKey " + encoded)); + return client().performRequest(request); + } + + private void setupQueryClusterRCS1(boolean useProxyMode) throws IOException { final Settings.Builder builder = Settings.builder(); if (useProxyMode) { builder.put("cluster.remote.my_remote_cluster.mode", "proxy") @@ -252,14 +239,37 @@ private void setupQueryClusterRemoteClusters(boolean useProxyMode) throws IOExce updateClusterSettings(builder.build()); } - private Response performRequestWithRemoteAccessUser(final Request request) throws IOException { - request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue(REMOTE_SEARCH_USER, PASS))); - return client().performRequest(request); + private Request esqlRequest(String command) throws IOException { + XContentBuilder body = JsonXContent.contentBuilder(); + body.startObject(); + body.field("query", command); + body.field("include_ccs_metadata", true); + body.endObject(); + Request request = new Request("POST", "_query"); + request.setJsonEntity(org.elasticsearch.common.Strings.toString(body)); + return request; } - private Response performRequestWithApiKey(final Request request, final String encoded) throws IOException { - request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", "ApiKey " + encoded)); - return client().performRequest(request); + private void assertSearchResponse(SearchResponse searchResponse, boolean alsoSearchLocally) { + try { + final List actualIndices = Arrays.stream(searchResponse.getHits().getHits()) + .map(SearchHit::getIndex) + .collect(Collectors.toList()); + if (alsoSearchLocally) { + assertThat(actualIndices, containsInAnyOrder("remote_index1", "local_index")); + } else { + assertThat(actualIndices, containsInAnyOrder("remote_index1")); + } + } finally { + searchResponse.decRef(); + } } + private void assertEsqlResponse(Response response) throws IOException { + assertOK(response); + String responseAsString = EntityUtils.toString(response.getEntity()); + assertThat(responseAsString, containsString("\"my_remote_cluster\":{\"status\":\"successful\"")); + assertThat(responseAsString, containsString("local_bar")); + assertThat(responseAsString, containsString("bar")); + } } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBWCToRCS1ClusterRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBWCToRCS1ClusterRestIT.java new file mode 100644 index 0000000000000..73e0f096039f9 --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBWCToRCS1ClusterRestIT.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.Version; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +/** + * BWC test which ensures that users and API keys with defined {@code remote_indices}/{@code remote_cluster} privileges can be used + * to query legacy remote clusters when using RCS 1.0. We send the request the to an older fulfilling cluster using RCS 1.0 with a user/role + * and API key where the {@code remote_indices}/{@code remote_cluster} are defined in the newer query cluster. + * All RCS 2.0 config should be effectively ignored when using RCS 1 for CCS. We send to an elder fulfil cluster to help ensure that + * newly introduced RCS 2.0 artifacts are forward compatible from the perspective of the old cluster. For example, a new privilege + * sent to an old cluster should be ignored. + */ +public class RemoteClusterSecurityBWCToRCS1ClusterRestIT extends AbstractRemoteClusterSecurityBWCRestIT { + + private static final Version OLD_CLUSTER_VERSION = Version.fromString(System.getProperty("tests.old_cluster_version")); + + static { + fulfillingCluster = ElasticsearchCluster.local() + .version(OLD_CLUSTER_VERSION) + .distribution(DistributionType.DEFAULT) + .name("fulfilling-cluster") + .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") + // .setting("logger.org.elasticsearch.xpack.core", "trace") //useful for human debugging + // .setting("logger.org.elasticsearch.xpack.security", "trace") //useful for human debugging + .build(); + + queryCluster = ElasticsearchCluster.local() + .version(Version.CURRENT) + .distribution(DistributionType.DEFAULT) + .setting("xpack.ml.enabled", "false") + .name("query-cluster") + .apply(commonClusterConfig) + .setting("xpack.security.remote_cluster_client.ssl.enabled", "true") + .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") + .rolesFile(Resource.fromClasspath("roles.yml")) + .build(); + } + + @ClassRule + // Use a RuleChain to ensure that fulfilling cluster is started before query cluster + public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster); + + @Override + protected boolean isRCS2() { + return false; + } + + @Before + @Override + public void setUp() throws Exception { + configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, true, randomBoolean(), false); + super.setUp(); + } +} diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBWCToRCS2ClusterRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBWCToRCS2ClusterRestIT.java new file mode 100644 index 0000000000000..5e173b72c66de --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityBWCToRCS2ClusterRestIT.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.Version; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +/** + * BWC test which ensures that users and API keys with defined {@code remote_indices}/{@code remote_cluster} privileges can be used + * to query older remote clusters when using RCS 2.0. We send the request the to an older fulfilling cluster using RCS 2.0 with a user/role + * and API key where the {@code remote_indices}/{@code remote_cluster} are defined in the newer query cluster. + * All new RCS 2.0 config should be effectively ignored when sending to older RCS 2.0. For example, a new privilege + * sent to an old cluster should be ignored. + */ +public class RemoteClusterSecurityBWCToRCS2ClusterRestIT extends AbstractRemoteClusterSecurityBWCRestIT { + + private static final Version OLD_CLUSTER_VERSION = Version.fromString(System.getProperty("tests.old_cluster_version")); + private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); + + static { + + fulfillingCluster = ElasticsearchCluster.local() + .name("fulfilling-cluster") + .version(OLD_CLUSTER_VERSION) + .distribution(DistributionType.DEFAULT) + .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") + .setting("remote_cluster_server.enabled", "true") + .setting("remote_cluster.port", "0") + .setting("xpack.security.remote_cluster_server.ssl.enabled", "true") + .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") + .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") + .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") + // .setting("logger.org.elasticsearch.xpack.core", "trace") //useful for human debugging + // .setting("logger.org.elasticsearch.xpack.security", "trace") //useful for human debugging + .build(); + + queryCluster = ElasticsearchCluster.local() + .name("query-cluster") + .distribution(DistributionType.DEFAULT) + .setting("xpack.ml.enabled", "false") + .apply(commonClusterConfig) + .setting("xpack.security.remote_cluster_client.ssl.enabled", "true") + .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") + .keystore("cluster.remote.my_remote_cluster.credentials", () -> { + if (API_KEY_MAP_REF.get() == null) { + final Map apiKeyMap = createCrossClusterAccessApiKey(""" + { + "search": [ + { + "names": ["*"] + } + ] + }"""); + API_KEY_MAP_REF.set(apiKeyMap); + } + return (String) API_KEY_MAP_REF.get().get("encoded"); + }) + .build(); + } + + @ClassRule + // Use a RuleChain to ensure that fulfilling cluster is started before query cluster + public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster); + + @Override + protected boolean isRCS2() { + return true; + } + + @Before + @Override + public void setUp() throws Exception { + configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, false, randomBoolean(), false); + super.setUp(); + } +} diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestStatsIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestStatsIT.java new file mode 100644 index 0000000000000..e98fcf6f72881 --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestStatsIT.java @@ -0,0 +1,266 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Strings; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchResponseUtils; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.junit.RunnableTestRuleAdapter; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.xcontent.XContentType; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; + +public class RemoteClusterSecurityRestStatsIT extends AbstractRemoteClusterSecurityTestCase { + + private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); + private static final AtomicReference> REST_API_KEY_MAP_REF = new AtomicReference<>(); + private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean(); + private static final AtomicBoolean NODE1_RCS_SERVER_ENABLED = new AtomicBoolean(); + private static final AtomicBoolean NODE2_RCS_SERVER_ENABLED = new AtomicBoolean(); + private static final int FULFILL_NODE_COUNT = 3; + private static final Logger logger = LogManager.getLogger(RemoteClusterSecurityRestStatsIT.class); + + static { + fulfillingCluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .name("fulfilling-cluster") + .nodes(FULFILL_NODE_COUNT) + .apply(commonClusterConfig) + .setting("remote_cluster.port", "0") + .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") + .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") + .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true")) + .node(1, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE1_RCS_SERVER_ENABLED.get()))) + .node(2, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE2_RCS_SERVER_ENABLED.get()))) + .build(); + + queryCluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .name("query-cluster") + .apply(commonClusterConfig) + .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("cluster.remote.my_remote_cluster.credentials", () -> { + if (API_KEY_MAP_REF.get() == null) { + final Map apiKeyMap = createCrossClusterAccessApiKey(""" + { + "search": [ + { + "names": ["*"] + } + ] + }"""); + API_KEY_MAP_REF.set(apiKeyMap); + } + return (String) API_KEY_MAP_REF.get().get("encoded"); + }) + // Define a bogus API key for another remote cluster + .keystore("cluster.remote.invalid_remote.credentials", randomEncodedApiKey()) + // Define remote with a REST API key to observe expected failure + .keystore("cluster.remote.wrong_api_key_type.credentials", () -> { + if (REST_API_KEY_MAP_REF.get() == null) { + initFulfillingClusterClient(); + final var createApiKeyRequest = new Request("POST", "/_security/api_key"); + createApiKeyRequest.setJsonEntity(""" + { + "name": "rest_api_key" + }"""); + try { + final Response createApiKeyResponse = performRequestWithAdminUser(fulfillingClusterClient, createApiKeyRequest); + assertOK(createApiKeyResponse); + REST_API_KEY_MAP_REF.set(responseAsMap(createApiKeyResponse)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return (String) REST_API_KEY_MAP_REF.get().get("encoded"); + }) + .rolesFile(Resource.fromClasspath("roles.yml")) + .user(REMOTE_METRIC_USER, PASS.toString(), "read_remote_shared_metrics", false) + .build(); + } + + @ClassRule + // Use a RuleChain to ensure that fulfilling cluster is started before query cluster + // `SSL_ENABLED_REF` is used to control the SSL-enabled setting on the test clusters + // We set it here, since randomization methods are not available in the static initialize context above + public static TestRule clusterRule = RuleChain.outerRule(new RunnableTestRuleAdapter(() -> { + SSL_ENABLED_REF.set(usually()); + NODE1_RCS_SERVER_ENABLED.set(randomBoolean()); + NODE2_RCS_SERVER_ENABLED.set(randomBoolean()); + })).around(fulfillingCluster).around(queryCluster); + + public void testCrossClusterStats() throws Exception { + configureRemoteCluster(); + setupRoleAndUserQueryCluster(); + addDocToIndexFulfillingCluster("index1"); + + // search #1 + searchFulfillingClusterFromQueryCluster("index1"); + Map statsResponseAsMap = getFulfillingClusterStatsFromQueryCluster(); + assertThat(ObjectPath.evaluate(statsResponseAsMap, "ccs.clusters.my_remote_cluster.nodes_count"), equalTo(FULFILL_NODE_COUNT)); + assertThat(ObjectPath.evaluate(statsResponseAsMap, "ccs._search.clusters.my_remote_cluster.total"), equalTo(1)); + int initialIndexCount = ObjectPath.evaluate(statsResponseAsMap, "ccs.clusters.my_remote_cluster.indices_count"); + + // search #2 + searchFulfillingClusterFromQueryCluster("index1"); + statsResponseAsMap = getFulfillingClusterStatsFromQueryCluster(); + assertThat(ObjectPath.evaluate(statsResponseAsMap, "ccs._search.total"), equalTo(2)); + assertThat(ObjectPath.evaluate(statsResponseAsMap, "ccs._search.clusters.my_remote_cluster.total"), equalTo(2)); + + // search #3 + expectThrows(Exception.class, () -> searchFulfillingClusterFromQueryCluster("junk")); + statsResponseAsMap = getFulfillingClusterStatsFromQueryCluster(); + assertThat(ObjectPath.evaluate(statsResponseAsMap, "ccs._search.total"), equalTo(3)); + assertThat(ObjectPath.evaluate(statsResponseAsMap, "ccs._search.clusters.my_remote_cluster.total"), equalTo(2)); + + // search #4 + addDocToIndexFulfillingCluster("index2"); + searchFulfillingClusterFromQueryCluster("index2"); + statsResponseAsMap = getFulfillingClusterStatsFromQueryCluster(); + assertThat(ObjectPath.evaluate(statsResponseAsMap, "ccs._search.total"), equalTo(4)); + assertThat(ObjectPath.evaluate(statsResponseAsMap, "ccs._search.clusters.my_remote_cluster.total"), equalTo(3)); + int updatedIndexCount = ObjectPath.evaluate(statsResponseAsMap, "ccs.clusters.my_remote_cluster.indices_count"); + assertThat(updatedIndexCount, equalTo(initialIndexCount + 1)); + } + + private Map getFulfillingClusterStatsFromQueryCluster() throws IOException { + return getFulfillingClusterStatsFromQueryCluster(false); + } + + private Map getFulfillingClusterStatsFromQueryCluster(boolean humanDebug) throws IOException { + Request stats = new Request("GET", "_cluster/stats?include_remotes=true&filter_path=ccs"); + Response statsResponse = performRequestWithRemoteSearchUser(stats); + if (humanDebug) { + debugResponse(statsResponse); + } + return entityAsMap(statsResponse.getEntity()); + } + + private void searchFulfillingClusterFromQueryCluster(String index, boolean humanDebug) throws IOException { + final var searchRequest = new Request( + "GET", + String.format( + Locale.ROOT, + "/%s:%s/_search?ccs_minimize_roundtrips=%s", + randomFrom("my_remote_cluster", "*", "my_remote_*"), + index, + randomBoolean() + ) + ); + Response response = performRequestWithRemoteSearchUser(searchRequest); + if (humanDebug) { + debugResponse(response); + } + assertOK(response); + final SearchResponse searchResponse = SearchResponseUtils.parseSearchResponse(responseAsParser(response)); + try { + final List actualIndices = Arrays.stream(searchResponse.getHits().getHits()) + .map(SearchHit::getIndex) + .collect(Collectors.toList()); + assertThat(actualIndices, containsInAnyOrder(index)); + + } finally { + searchResponse.decRef(); + } + } + + private void searchFulfillingClusterFromQueryCluster(String index) throws IOException { + searchFulfillingClusterFromQueryCluster(index, false); + } + + private void addDocToIndexFulfillingCluster(String index) throws IOException { + // Index some documents, so we can attempt to search them from the querying cluster + final Request bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(Strings.format(""" + { "index": { "_index": "%s" } } + { "foo": "bar" } + """, index)); + assertOK(performRequestAgainstFulfillingCluster(bulkRequest)); + } + + private void setupRoleAndUserQueryCluster() throws IOException { + final var putRoleRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE); + putRoleRequest.setJsonEntity(""" + { + "description": "Role with privileges for remote indices and stats.", + "cluster": ["monitor_stats"], + "remote_indices": [ + { + "names": ["*"], + "privileges": ["read", "read_cross_cluster"], + "clusters": ["*"] + } + ], + "remote_cluster": [ + { + "privileges": ["monitor_stats"], + "clusters": ["*"] + } + ] + }"""); + assertOK(adminClient().performRequest(putRoleRequest)); + final var putUserRequest = new Request("PUT", "/_security/user/" + REMOTE_SEARCH_USER); + putUserRequest.setJsonEntity(""" + { + "password": "x-pack-test-password", + "roles" : ["remote_search"] + }"""); + assertOK(adminClient().performRequest(putUserRequest)); + } + + private Response performRequestWithRemoteSearchUser(final Request request) throws IOException { + request.setOptions( + RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", headerFromRandomAuthMethod(REMOTE_SEARCH_USER, PASS)) + ); + return client().performRequest(request); + } + + // helper method for humans see the responses for debug purposes, when used will always fail the test + private void debugResponse(Response response) throws IOException { + String jsonString = XContentHelper.convertToJson( + new BytesArray(EntityUtils.toString(response.getEntity())), + true, + true, + XContentType.JSON + ); + logger.error(jsonString); + assertFalse(true); // boom + } +} diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java index 3dd678046ea5f..324850f158268 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java @@ -29,7 +29,6 @@ public class SecurityWithBasicLicenseIT extends SecurityInBasicRestTestCase { - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99169") public void testWithBasicLicense() throws Exception { checkLicenseType("basic"); checkSecurityEnabled(false); diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 667140b849951..8ce7fc77fe4f3 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -828,7 +828,7 @@ public void testRemoteClusterSupportForApiKeys() throws IOException { assertOK(response); assertAPIKeyWithRemoteClusterPermissions(apiKeyId, includeRemoteCluster, false, null, new String[] { "foo", "bar" }); - // create API key as the remote user which does remote_cluster limited_by permissions + // create API key as the remote user which has all remote_cluster permissions via limited_by response = sendRequestAsRemoteUser(createApiKeyRequest); apiKeyId = ObjectPath.createFromResponse(response).evaluate("id"); assertThat(apiKeyId, notNullValue()); @@ -922,7 +922,7 @@ private void assertAPIKeyWithRemoteClusterPermissions( assertNotNull(limitedByRole); List>> remoteCluster = (List>>) limitedByRole.get("remote_cluster"); - assertThat(remoteCluster.get(0).get("privileges"), containsInAnyOrder("monitor_enrich")); + assertThat(remoteCluster.get(0).get("privileges"), containsInAnyOrder("monitor_stats", "monitor_enrich")); assertThat(remoteCluster.get(0).get("clusters"), containsInAnyOrder("remote")); } else { // no limited by permissions diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java index d9778fda6e486..d421f63ed2a02 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java @@ -512,7 +512,7 @@ public static void buildRoleFromDescriptors( ); }); - if (remoteClusterPermissions.hasPrivileges()) { + if (remoteClusterPermissions.hasAnyPrivileges()) { builder.addRemoteClusterPermissions(remoteClusterPermissions); } else { builder.addRemoteClusterPermissions(RemoteClusterPermissions.NONE); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleDescriptorStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleDescriptorStore.java index ac8d84d95fd1d..a64cef366926f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleDescriptorStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleDescriptorStore.java @@ -150,7 +150,7 @@ public void resolveCrossClusterAccessRoleReference( + "but other privileges found for subject [" + crossClusterAccessRoleReference.getUserPrincipal() + "]"; - logger.debug("{}. Invalid role descriptor: [{}]", message, roleDescriptor); + logger.warn("{}. Invalid role descriptor: [{}]", message, roleDescriptor); listener.onFailure(new IllegalArgumentException(message)); return; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java index d71c2b0d19074..a41c54ada781a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java @@ -92,6 +92,7 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeTests; +import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver; import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges.ManageApplicationPrivileges; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.Privilege; @@ -1312,10 +1313,7 @@ public void testBuildUserPrivilegeResponse() { ) .addRemoteClusterPermissions( new RemoteClusterPermissions().addGroup( - new RemoteClusterPermissionGroup( - RemoteClusterPermissions.getSupportedRemoteClusterPermissions().toArray(new String[0]), - new String[] { "remote-1" } - ) + new RemoteClusterPermissionGroup(new String[] { "monitor_enrich" }, new String[] { "remote-1" }) ) .addGroup( new RemoteClusterPermissionGroup( @@ -1383,26 +1381,33 @@ public void testBuildUserPrivilegeResponse() { RemoteClusterPermissions remoteClusterPermissions = response.getRemoteClusterPermissions(); String[] allRemoteClusterPermissions = RemoteClusterPermissions.getSupportedRemoteClusterPermissions().toArray(new String[0]); - assert allRemoteClusterPermissions.length == 1 - : "if more remote cluster permissions are added this test needs to be updated to ensure the correct remotes receive the " - + "correct permissions. "; - // 2 groups with 3 aliases + assertThat(response.getRemoteClusterPermissions().groups(), iterableWithSize(2)); - assertEquals( - 3, - response.getRemoteClusterPermissions() - .groups() - .stream() - .map(RemoteClusterPermissionGroup::remoteClusterAliases) - .flatMap(Arrays::stream) - .distinct() - .count() + // remote-1 has monitor_enrich permission + // remote-2 and remote-3 have all permissions + assertThat( + response.getRemoteClusterPermissions().groups(), + containsInAnyOrder( + new RemoteClusterPermissionGroup(new String[] { "monitor_enrich" }, new String[] { "remote-1" }), + new RemoteClusterPermissionGroup(allRemoteClusterPermissions, new String[] { "remote-2", "remote-3" }) + ) + ); + + // ensure that all permissions are valid for the current transport version + assertThat( + Arrays.asList(remoteClusterPermissions.collapseAndRemoveUnsupportedPrivileges("remote-1", TransportVersion.current())), + hasItem("monitor_enrich") ); for (String permission : RemoteClusterPermissions.getSupportedRemoteClusterPermissions()) { - assertThat(Arrays.asList(remoteClusterPermissions.privilegeNames("remote-1", TransportVersion.current())), hasItem(permission)); - assertThat(Arrays.asList(remoteClusterPermissions.privilegeNames("remote-2", TransportVersion.current())), hasItem(permission)); - assertThat(Arrays.asList(remoteClusterPermissions.privilegeNames("remote-3", TransportVersion.current())), hasItem(permission)); + assertThat( + Arrays.asList(remoteClusterPermissions.collapseAndRemoveUnsupportedPrivileges("remote-2", TransportVersion.current())), + hasItem(permission) + ); + assertThat( + Arrays.asList(remoteClusterPermissions.collapseAndRemoveUnsupportedPrivileges("remote-3", TransportVersion.current())), + hasItem(permission) + ); } } @@ -1782,7 +1787,10 @@ public void testGetRoleDescriptorsForRemoteClusterForReservedRoles() { new RoleDescriptorsIntersection( new RoleDescriptor( Role.REMOTE_USER_ROLE_NAME, - null, + RemoteClusterPermissions.getSupportedRemoteClusterPermissions() + .stream() + .filter(s -> s.equals(ClusterPrivilegeResolver.MONITOR_STATS.name())) + .toArray(String[]::new), new IndicesPrivileges[] { IndicesPrivileges.builder().indices(".monitoring-*").privileges("read", "read_cross_cluster").build(), IndicesPrivileges.builder().indices("apm-*").privileges("read", "read_cross_cluster").build(), diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 1a454a18e437a..ea7257f64c82b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -1077,7 +1077,7 @@ public ClusterPermission.Builder buildPermission(ClusterPermission.Builder build assertHasRemoteIndexGroupsForClusters(forRemote, Set.of("*"), indexGroup("remote-idx-2-*")); assertValidRemoteClusterPermissions(role.remoteCluster(), new String[] { "remote-*" }); assertThat( - role.remoteCluster().privilegeNames("remote-foobar", TransportVersion.current()), + role.remoteCluster().collapseAndRemoveUnsupportedPrivileges("remote-foobar", TransportVersion.current()), equalTo(RemoteClusterPermissions.getSupportedRemoteClusterPermissions().toArray(new String[0])) ); } @@ -3196,12 +3196,12 @@ private void assertValidRemoteClusterPermissions(RemoteClusterPermissions permis } private void assertValidRemoteClusterPermissionsParent(RemoteClusterPermissions permissions, String[] aliases) { - assertTrue(permissions.hasPrivileges()); + assertTrue(permissions.hasAnyPrivileges()); for (String alias : aliases) { - assertTrue(permissions.hasPrivileges(alias)); - assertFalse(permissions.hasPrivileges(randomValueOtherThan(alias, () -> randomAlphaOfLength(5)))); + assertTrue(permissions.hasAnyPrivileges(alias)); + assertFalse(permissions.hasAnyPrivileges(randomValueOtherThan(alias, () -> randomAlphaOfLength(5)))); assertThat( - permissions.privilegeNames(alias, TransportVersion.current()), + permissions.collapseAndRemoveUnsupportedPrivileges(alias, TransportVersion.current()), arrayContaining(RemoteClusterPermissions.getSupportedRemoteClusterPermissions().toArray(new String[0])) ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java index 0a2c40d2a257a..fc96c247b2aba 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/FileRolesStoreTests.java @@ -388,7 +388,8 @@ public void testParseFileWithRemoteIndicesAndCluster() throws IllegalAccessExcep events.get(4), startsWith( "failed to parse remote_cluster for role [invalid_role_bad_priv_remote_clusters]. " - + "[monitor_enrich] is the only value allowed for [privileges] within [remote_cluster]. skipping role..." + + "[monitor_enrich, monitor_stats] are the only values allowed for [privileges] within [remote_cluster]. " + + "Found [junk]. skipping role..." ) ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesActionTests.java index e17d651a19748..5b91b774cc435 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesActionTests.java @@ -213,7 +213,7 @@ public void testBuildResponse() throws Exception { ,"remote_cluster":[ { "privileges":[ - "monitor_enrich" + "monitor_enrich", "monitor_stats" ], "clusters":[ "remote-1" @@ -221,7 +221,7 @@ public void testBuildResponse() throws Exception { }, { "privileges":[ - "monitor_enrich" + "monitor_enrich", "monitor_stats" ], "clusters":[ "remote-2", diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/10_basic.yml index c9b05c4e13a85..17e5e0cfb0759 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/enrich/10_basic.yml @@ -69,7 +69,7 @@ setup: - do: warnings: - - "the [elasticsearch_version] field of an enrich policy has no effect and will be removed in Elasticsearch 9.0" + - "the [elasticsearch_version] field of an enrich policy has no effect and will be removed in a future version of Elasticsearch" enrich.put_policy: name: policy-crud-warning body: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/privileges/11_builtin.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/privileges/11_builtin.yml index ef8fab9ca7b6d..d03e6925cab1f 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/privileges/11_builtin.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/privileges/11_builtin.yml @@ -15,5 +15,5 @@ setup: # This is fragile - it needs to be updated every time we add a new cluster/index privilege # I would much prefer we could just check that specific entries are in the array, but we don't have # an assertion for that - - length: { "cluster" : 61 } + - length: { "cluster" : 62 } - length: { "index" : 22 }